import math
import random
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from itertools import count
import numpy as np
from pdb import set_trace

import sys
sys.path.append('..')
from env.atari_task import Pendulum,MountainCarContinuous,BipedalWalker
from core.memory import ReplayMemory,Transition
from core.model_ac import ActorNet,CriticNet
from core.policy_greedy import GreedyPolicy
from core.random_process import OrnsteinUhlenbeckProcess
#from tf_logger import Logger
from utils.tools import *
from utils import plot

class DQNAgent:
    def __init__(self):
        self.steps_done = 0
        self.batch_size = 64
        self.GAMMA= 0.99
        self.num_episodes = 2000
        self.task = BipedalWalker()
        self.memory = ReplayMemory(10000,self.batch_size)
        self.eval_actornet = ActorNet(24,4)
        self.target_actornet = ActorNet(24,4)
        self.target_actornet.load_state_dict(self.eval_actornet.state_dict())
        self.eval_criticnet =  CriticNet(24,4)
        self.target_criticnet = CriticNet(24,4)
        self.target_criticnet.load_state_dict(self.eval_criticnet.state_dict())
        self.actor_optimizer = torch.optim.Adam(self.eval_actornet.parameters(), lr=1e-4)
        self.critic_optimizer = torch.optim.Adam(self.eval_criticnet.parameters(), lr=1e-3,weight_decay=0.01)
        self.random_process = OrnsteinUhlenbeckProcess(theta=0.15,sigma=0.2)
        self.epsilon = 1
        self.epsilon_d = 1/30000
        self.criterion = torch.nn.MSELoss()
        #self.policy = GreedyPolicy(EPS_START = 0.9,EPS_END = 0.05,EPS_DECAY = 200)
        self.episode_durations= []

    def select_action(self,state):
        action = to_numpy(self.eval_actornet(Variable(state).type(torch.FloatTensor)))

        if self.steps_done % 50 == 0:
            print(action,to_numpy(self.eval_criticnet(Variable(state).type(torch.FloatTensor),to_tensor(action))),max(self.epsilon,0) * self.random_process.sample()*2)
        action += max(self.epsilon,0) * self.random_process.sample()
        self.epsilon -= self.epsilon_d
        self.steps_done += 1
        #set_trace()
        return action

    def optimize_model(self):
        transitions = self.memory.sample()
        if not transitions:
            return

        batch = Transition(*zip(*transitions))
        state_batch = Variable(torch.cat(batch.state)).type(torch.FloatTensor)
        action_batch = Variable(torch.cat(batch.action)).type(torch.FloatTensor)
        reward_batch = Variable(torch.cat(batch.reward)).type(torch.FloatTensor)
        next_state_batch = Variable(torch.cat(batch.next_state)).type(torch.FloatTensor)
        done = Variable(torch.cat(batch.done)).type(torch.FloatTensor)
        #print(reward_batch)
        #set_trace()
        q_next=self.target_criticnet(next_state_batch,self.target_actornet(next_state_batch))
        q_next = self.GAMMA * q_next * (1 - done)
        q_next.add_(reward_batch)
        q_next=Variable(q_next.data)
        q_ = self.eval_criticnet(state_batch,action_batch)
        critic_loss = self.criterion(q_, q_next)
        self.critic_optimizer.zero_grad()
        critic_loss.backward()
        #torch.nn.utils.clip_grad_norm(self.model.parameters(), 1)
        self.critic_optimizer.step()

        actor_loss = -self.eval_criticnet(state_batch, self.eval_actornet(state_batch))
        actor_loss = actor_loss.mean()
        self.actor_optimizer.zero_grad()
        actor_loss.backward()
        self.actor_optimizer.step()

        #set_trace()
        soft_target_model_updates(self.target_actornet,self.eval_actornet,0.001)
        soft_target_model_updates(self.target_criticnet,self.eval_criticnet,0.001)

    def episode(self):
        episode_num=[]
        episode_durations=[]
        episode_temp=[]
        self.random_process.reset_states()
        for i_episode in range(self.num_episodes):
            state = torch.from_numpy(self.task.reset()).unsqueeze(0)
            for t in count():
                #set_trace()
                self.task.env.render()
                action = self.select_action(state)#.unsqueeze(0))
                next_state, reward, done, _ = self.task.step(action[0])
                #set_trace()

                #reward = reward +next_state[0]
                #print(reward)
                action = torch.from_numpy(action)#.unsqueeze(0)
                next_state = torch.from_numpy(next_state).unsqueeze(0)
                reward_ = torch.FloatTensor([reward]).unsqueeze(0)
                done_ = torch.from_numpy(np.array([int(done)])).unsqueeze(0)

                self.memory.push(state,action,next_state,reward_, done_)
                state = next_state
                self.optimize_model()

                episode_temp.append(reward)

                if (self.steps_done ) % 100 == 0:
                    episode_num.append(self.steps_done + 1)
                    episode_durations.append(episode_temp)
                    episode_temp = []
                    plot.plot_line(episode_num,episode_durations)

                if done  or t > 300:
                    break

            if self.steps_done>30000:
                return 1

        self.task.env.render(close=True)
        self.task.env.close()

#if __name__ == '__main__':
a= DQNAgent()
a.episode()


import datetime, Activity

class Vehicle:
    def __init__(self, id1, canTake, start, end, capacity, availability):
        """
        Parameters:
            “id” which contains the id of the vehicle.
            “canTake” which is a set of categories of patients that the vehicle can take.
            “start” and “end” which are the ids of the starting and ending depots of the vehicle. A -1 value indicates that the vehicle has no depot.
            “max_capacity” which is the capacity of the vehicle.
            “availability” which is a set of time windows when the vehicle is available. Each time window is encoded by a string having the following format: “HHhMM:HHhMM”.
            "history" list of activities
        """
        self.id = id1
        self.canTake = canTake
        self.start = start
        self.end = end
        self.max_capacity = capacity
        self.availability = availability
        self.history = list()

    def setActivity(self,act):
        self.history.append(act)

    def deleteActivity(self,act):
        self.history.remove(act)

    def __eq__(self,other):
        if other == None:
            return False
        if self == None:
            return False
        return self.id == other.id and self.canTake == other.canTake and self.start == other.start and self.end == other.end and self.max_capacity == other.max_capacity and self.availability == other.availability

    def getLastAct(self,time):
        history_sorted = sorted(self.history,key=lambda y:y.time)
        i = 0
        if len(history_sorted)==0:
            return Activity.Activity(self.start,self.getTimeWindow()[0],-1,0,0)
        while i<len(history_sorted) and history_sorted[i].time<time :
            i+=1
        if i==len(history_sorted):
            return history_sorted[-1]
        return history_sorted[i-1]

    def getTimeWindow(self):
        windows = []
        for it in self.availability:
            start, end = it.split(":")
            startH, startM = start.split("h")
            endH, endM = end.split("h")
            st = datetime.timedelta(hours=int(startH), minutes=int(startM))
            en = datetime.timedelta(hours=int(endH), minutes=int(endM))
            windows.append((st, en))
        return windows


import pandas as pd
import numpy as np

train = pd.read_csv("train.csv")
test = pd.read_csv("test.csv")
train_data = train.values[:, 1:]
train_label = train.values[:, 0]
test_data = test.values
print(test_data[:20])
print(train_data.shape)
print(train_label[:25])


# Copyright (C) 2014,2017 Andreas Lang-Nevyjel, init.at
#
# Send feedback to: <lang-nevyjel@init.at>
#
# this file is part of icsw-server
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License Version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
""" SNMP result functions """

def oid_to_str(oid):
    if isinstance(oid, tuple):
        oid = ".".join(
            [
                "{:d}".format(_p) for _p in oid
            ]
        )
    return oid

def str_to_oid(oid):
    if isinstance(oid, str):
        oid = tuple(
            [
                int(_val) for _val in oid.split(".")
            ]
        )
    return oid

def dict_to_str_keys(in_dict):
    return {
        oid_to_str(_key): _value for _key, _value in in_dict.items()
    }

def simplify_dict(in_dict, start_tuple, sub_key_filter=[], keys_to_string=False):

    def _key_to_string(in_key):
        if isinstance(in_key, tuple):
            # interpret in_key als special encoded string (len, [chars, ...])
            l_key = list(in_key)
            _key = []
            while l_key:
                _key.append(
                    "".join(
                        [
                            chr(l_key.pop(0)) for _idx in range(l_key.pop(0))
                        ]
                    )
                )
            return tuple(_key)
        else:
            return in_key
    # check in_dict for all keys starting with start_tuple and return a reordered dict for faster
    # processing
    _slist = list(start_tuple)
    # get all valid keys
    _ref_keys = set(
        [
            _key for _key in in_dict.keys() if list(_key)[:len(start_tuple)] == _slist
        ]
    )
    # the last entry is the reference idx
    _result = {}
    if sub_key_filter:
        for _key in _ref_keys:
            _s_key = _key[len(start_tuple):]
            if _s_key[0] in sub_key_filter:
                if len(_s_key) == 2:
                    # integer as key
                    _result.setdefault(_s_key[1], {})[_s_key[0]] = in_dict[_key]
                elif _s_key:
                    # tuple as key
                    _result.setdefault(tuple(list(_s_key[1:])), {})[_s_key[0]] = in_dict[_key]
    else:
        for _key in _ref_keys:
            _s_key = _key[len(start_tuple):]
            if len(_s_key) == 2:
                # integer as key
                _result.setdefault(_s_key[1], {})[_s_key[0]] = in_dict[_key]
            elif _s_key:
                # tuple as key
                _result.setdefault(tuple(list(_s_key[1:])), {})[_s_key[0]] = in_dict[_key]
            else:
                _result[None] = in_dict[_key]
    if keys_to_string:
        _result = {
            _key_to_string(_key): _value for _key, _value in _result.items()
        }
    return _result

def reorder_dict(in_dict):
    # take in dict an reorder it using the following method:
    # in_dict: {key:value} where key has the form (idx, key[::])
    r_dict = {}
    for key, value in in_dict.items():
        print("***", key[:])
        r_dict.setdefault(key[1:], {})[key[0]] = value
    return r_dict

def flatten_dict(in_dict):
    _changed = True
    while _changed:
        _changed = False
        if len(in_dict) == 1:
            in_dict = list(in_dict.values())[0]
        _r_dict = {}
        for _key, _value in in_dict.items():
            if isinstance(_value, dict):
                _changed = True
                if isinstance(_key, int):
                    _key = [_key]
                else:
                    _key = list(_key)
                for _s_key, _s_value in _value.items():
                    if isinstance(_s_key, int):
                        _s_key = _key + [_s_key]
                    else:
                        _s_key = _key + list(_s_key)
                    _r_dict[tuple(_s_key)] = _s_value
            else:
                _r_dict[_key] = _value
        in_dict = _r_dict
    return _r_dict


'''
Alpha Perceptron (Frank Rosenblat i think)
'''
import numpy as np

# Settings
num_epochs = 100

# Input Matrix (last column is the class label)
# data = np.array([[1, 1, 1, 1], # this dataset is impossible for a perceptron to learn
# 				 [0, 0, 1, 1],
# 				 [1, 0, 1, 0],
# 				 [0, 1, 1, 0]])

data = np.array([[.5, -.5, 1, 1],
				 [.7, -.7, 1, 0]])

# Weight Matrix
w = np.array([[2, 1, .5]])

for epoch in range(num_epochs):
	for index, row in enumerate(data):
		x = data[index,:-1]
		target = data[index,-1] # get the class label
		output = np.sum(x.T * w) # multiple weights by input
		if output >= 0:
			output = 1
		else:
			output = 0

		if output == target:
			print(output, 'correct')
		else:
			if output == 1:
				print(output, 'incorrect +')
				w = w - x
			elif output == 0:
				print(output, 'incorrect -')
				w = w + x

print(w, output)


import pickle
import gzip
from scipy.sparse import hstack
from keras.models import Model, load_model
import sys

def classification(classifier, dic, text):
	'''Args:
		classifier: The best model trained on the entire data
		dic: A dictionary storing the Vocabulary, word level tf-idf object, ngram level tf-idf object, characters level tf-idf object and label encoder
		text: input text whose label is to be predicted
	Returns:
		Predicted Label of the input_text'''

	Vocabulary=dic["Vocabulary"]
	tfidf_vect=dic["tfidf_vect"] # word level tf-idf
	tfidf_vect_ngram=dic["tfidf_vect_ngram"] # ngram level tf-idf
	tfidf_vect_ngram_chars=dic["tfidf_vect_ngram_chars"] # characters level tf-idf
	encoder=dic["encoder"] #label encoder
	# replacing out of vocabulary words with OOV
	for word in text.split():
		if word not in Vocabulary:
			text=text.replace(word, "OOV")
	text=[text]
	x_tfidf =  tfidf_vect.transform(text)
	x_tfidf_ngram =  tfidf_vect_ngram.transform(text)
	x_tfidf_ngram_chars =  tfidf_vect_ngram_chars.transform(text)
	# Combining Word Level TF IDF Vectors, Ngram Level TF IDF Vectors and Character Level TF IDF Vectors
	X=hstack([x_tfidf, x_tfidf_ngram, x_tfidf_ngram_chars]).toarray()
	Y=classifier.predict(X)
	Y=Y.argmax(axis=-1)
	Y=encoder.inverse_transform(Y)
	return (Y[0])

def main():
	text_for_prediction=sys.argv[1]
	# reading the files
	path_to_model="./"
	f = gzip.open(path_to_model+'dictionary.pklz','rb')
	dic = pickle.load(f)
	f.close()
	classifier = load_model('my_trained_model.h5')
	print ("reading done")
	print ("The label:\t"+classification(classifier, dic, text_for_prediction))

if __name__=='__main__' :
    main()


# coding=utf-8
#
#  Copyright 2014-2016 F5 Networks Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""BIG-IP® Local Traffic Manager (LTM) policy module.

REST URI
    ``http://localhost/mgmt/tm/ltm/policy``

GUI Path
    ``Local Traffic --> policy``

REST Kind
    ``tm:ltm:policy:*``
"""

from f5.bigip.resource import Collection
from f5.bigip.resource import Resource

class Policys(Collection):
    """BIG-IP® LTM policy collection."""
    def __init__(self, ltm):
        super(Policys, self).__init__(ltm)
        self._meta_data['allowed_lazy_attributes'] = [Policy]
        self._meta_data['attribute_registry'] =\
            {'tm:ltm:policy:policystate': Policy}

class Policy(Resource):
    """BIG-IP® LTM policy resource."""
    def __init__(self, policy_s):
        super(Policy, self).__init__(policy_s)
        self._meta_data['required_json_kind'] = 'tm:ltm:policy:policystate'
        self._meta_data['required_creation_parameters'].update(('strategy',))
        temp = {'tm:ltm:policy:rules:rulescollectionstate': Rules_s}
        self._meta_data['attribute_registry'] = temp

class Rules_s(Collection):
    """BIG-IP® LTM policy rules sub-collection."""
    def __init__(self, policy):
        super(Rules_s, self).__init__(policy)
        self._meta_data['attribute_registry'] =\
            {'tm:ltm:policy:rules:rulesstate': Rules}
        self._meta_data['required_json_kind'] =\
            'tm:ltm:policy:rules:rulescollectionstate'
        self._meta_data['allowed_lazy_attributes'] = [Rules]

class Rules(Resource):
    """BIG-IP® LTM policy rules sub-collection resource."""
    def __init__(self, rules_s):
        super(Rules, self).__init__(rules_s)
        self._meta_data['required_json_kind'] =\
            'tm:ltm:policy:rules:rulesstate'
        temp = {'tm:ltm:policy:rules:actions:actionscollectionstate':
                Actions_s,
                'tm:ltm:policy:rules:conditions:conditionscollectionstate':
                Conditions_s}
        self._meta_data['attribute_registry'] = temp

class Actions_s(Collection):
    """BIG-IP® LTM policy actions sub-collection."""
    def __init__(self, rules):
        super(Actions_s, self).__init__(rules)
        self._meta_data['required_json_kind'] =\
            'tm:ltm:policy:rules:actions:actionscollectionstate'
        self._meta_data['allowed_lazy_attributes'] = [Actions]
        self._meta_data['attribute_registry'] =\
            {'tm:ltm:policy:rules:actions:actionsstate': Actions}

class Actions(Resource):
    """BIG-IP® LTM policy actions sub-collection resource."""
    def __init__(self, actions_s):
        super(Actions, self).__init__(actions_s)
        self._meta_data['required_json_kind'] =\
            'tm:ltm:policy:rules:actions:actionsstate'

class Conditions_s(Collection):
    """BIG-IP® LTM policy conditions sub-collection."""
    def __init__(self, rules):
        super(Conditions_s, self).__init__(rules)
        self._meta_data['required_json_kind'] =\
            'tm:ltm:policy:rules:conditions:conditionscollectionstate'
        self._meta_data['allowed_lazy_attributes'] = [Conditions]
        self._meta_data['attribute_registry'] =\
            {'tm:ltm:policy:rules:conditions:conditionsstate': Conditions}

class Conditions(Resource):
    """BIG-IP® LTM policy conditions sub-collection resource."""
    def __init__(self, conditions_s):
        super(Conditions, self).__init__(conditions_s)
        self._meta_data['required_json_kind'] =\
            'tm:ltm:policy:rules:conditions:conditionsstate'


import sys
import numpy
import scipy
import PIL

## Attempt to quantify the difference between two images

#Things to check:
#- Images of the same size and dimension
#- Alignment
#- Exposure and contrast of the images
#- Colour Information
#    - Is RGB a necessity, or will greyscale work?
#- Likeliness of the edges moving
#    - Can apply some kind of Edge Detection Algorithm, using Prewitt Transform
#- Noise
#- What kind of difference we're considering:
#    - Manhattan Norm (sum of absolute values) will tell how much the image is off.
#    - Zero norm (the number of elements not equal to 0) will tell how many pixels differ.

def compare_images(img1, img2):
    # Exposure difference could result in problems
    # Normalize first
    img1 = normalize(img1)
    img2 = normalize(img2)
    # Calculate the difference and its normalization
    diff = img1 - img2 # Element by element iteration
    m_norm = sum(abs(diff))
    z_norm = numpy.linalg.norm(diff.ravel(), 0)
    return (m_norm, z_norm)

def to_grayscale(arr):
    if len(arr.shape) == 3:
        return numpy.average(arr, -1) #average over the last color  channel axis
    else:
        return arr

def normalize(arr):
    rng = arr.max()-arr.min()
    amin = arr.min()
    return (arr-amin)*255/rng

if __name__ == "__main__":
    main()

def main(file1, file2):
    file1= file2 = sys.argv[1:1+2]
    # Use equivalent of MATLAB's imread function
    img1 = to_grayscale(scipy.misc.imread(file1).astype(float))
    img2 = to_grayscale(scipy.misc.imread(file2).astype(float))

    # Compare both images
    manhattan_norm, zero_norm = compare_images(img1, img2)
    print("Manhattan norm:", manhattan_norm, "/ per pixel: ", manhattan_norm/img1.size)
    print("Zero norm:", zero_norm, "/ per pixel: ", zero_norm/img1.size )


#!/usr/bin/python3
#/usr/local/bin/python3
import hmac
from requests import Request, Session
import datetime, time
import sys
import os
import pandas as pd
root_path = os.path.dirname(os.path.realpath(__file__)) + '/../'
sys.path.append(root_path + 'scripts/')
import utils

if len(sys.argv) < 2:
    print("usage: ./dump_ftx_future.py date")
    quit()
dt = sys.argv[1]
yy = int(utils.get_year(dt))
mm = int(utils.get_month(dt))
dd = int(utils.get_day(dt))

sd = datetime.datetime(yy, mm, dd, 0, 0)
ed = sd + datetime.timedelta(1)
print('{} {}'.format(sd, ed))
sd = int(time.mktime(sd.timetuple()))
ed = int(time.mktime(ed.timetuple()))
rics = pd.read_csv(root_path + '/data/compo/crypto/ftx.txt')
rics = list(rics.ric.unique())
erd_path = root_path + 'data/erd/crypto/'

s = Session()
ts = int(time.time() * 1000)
df = pd.DataFrame()
for ric in rics:
    request = Request('GET', 'https://ftx.com/api/markets/{}-PERP/candles?resolution=86400&start_time={}&end_time={}'.format(ric, sd, ed))
    prepared = request.prepare()
    signature_payload = f'{ts}{prepared.method}{prepared.path_url}'.encode()
    signature = hmac.new('mkW7nF6mjcqVR_HwD0f_6TuDirQZO_x_aJ-wW4f9'.encode(), signature_payload, 'sha256').hexdigest()
    prepared.headers['FTX-KEY'] = 'zve2arEfGvNGNWnooNwvf-7PiMvEEnl5gT_7PzTQ'
    prepared.headers['FTX-SIGN'] = signature
    prepared.headers['FTX-TS'] = str(ts)
    res = s.send(prepared).json()['result']
    if len(res) == 0: continue
    else: res = res[0]
    current_day = res['startTime'].split('T')[0]
    df = df.append({ 'ric': ric, 'date': current_day, 'open': res['open'], 'close': res['close'], 'high': res['high'], 'low': res['low'], 'pre_close': res['open'], 'volume': res['volume'] }, ignore_index = True)
cols = ['ric', 'date', 'open', 'close', 'high', 'low', 'pre_close', 'volume']
df = df[cols]
#df['date'] = utils.date_str(dt)
erd_file = erd_path + '/{}.txt'.format(dt)
print(erd_file)
df.to_csv(erd_file, sep='\t', index = False)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from egnyte import exc

from egnyte.tests.config import IntegrationCase

class TestFolders(IntegrationCase):
    def setUp(self):
        super(TestFolders, self).setUp()
        self.folder = self.root_folder.folder('Iñtërnâtiônàlizætiøν☃ test')
        self.dest = self.root_folder.folder('2')
        self.file = self.folder.file('test.txt')

    def tearDown(self):
        try:
            self.folder.delete()
        except exc.NotFound:
            pass
        try:
            self.dest.delete()
        except exc.NotFound:
            pass
        super(TestFolders, self).tearDown()

    def test_folder(self):
        self.folder.create()
        with self.assertRaises(exc.InsufficientPermissions):
            self.folder.create(False)
        self.folder.delete()
        with self.assertRaises(exc.NotFound):
            self.folder.delete()

    def test_folder_move(self):
        self.folder.create()
        moved = self.folder.move(self.dest.path)
        self.assertEqual(moved.path, self.dest.path, "Moved folder path should be identical")
        with self.assertRaises(exc.NotFound):
            self.folder.delete()
        self.dest.delete()

    def test_folder_copy(self):
        self.folder.create()
        copied = self.folder.copy(self.dest.path)
        self.assertEqual(copied.path, self.dest.path, "Copied folder path should be identical")
        self.folder.delete()
        self.dest.delete()

    def test_folder_list(self):
        folder = self.client.folder(self.folder.path).create()
        subfolder = folder.folder("test1").create()
        file = folder.file("test2")
        file.upload(b"test111")
        folder.list()
        folders = list(folder.folders)
        files = list(folder.files)
        self.assertEqual(1, len(folders), "There should be one subfolder")
        self.assertEqual(folders[0]._url, subfolder._url, "Subfolder URLs should be identical")
        self.assertEqual(1, len(files), "There should be one filer")
        self.assertEqual(files[0]._url, file._url, "File URLs should be identical")
        folder.delete()


from flask import g
from influenceexplorer import InfluenceExplorer
from sunlightapi import sunlight
import json
import requests

from capitolphone import settings

TITLES = {
    'Rep': 'Representative',
    'Sen': 'Senator',
}

sunlight.apikey = settings.SUNLIGHT_KEY
ie = InfluenceExplorer(settings.SUNLIGHT_KEY)

def load_call(sid, params):
    """ Loads a call from the datastore or creates a new one if one
        does not exist. Appends the current call status to the list
        of requests involved in this call.

        sid: the unique call ID from Twilio
        params: the POSTed request parameters
    """

    # find existing call
    doc = g.db.calls.find_one({'call_sid': sid})

    if doc is None:
        # create new call if call does not exist
        doc = {
            'call_sid': sid,
            'from': params['From'],
            'to': params['To'],
            'caller_name': params.get('CallerName', None),
            'context': {
                'zipcode': None,
                'legislator': None,
            },
        }
        g.db.calls.insert(doc)

    # create array for requests list
    if 'requests' not in doc:
        doc['requests'] = []

    # append current request information and update current status
    doc['requests'].append({
        'timestamp': g.now,
        'call_status': params['CallStatus']
    })
    doc['current_status'] = params['CallStatus']

    return doc

def legislators_for_zip(zipcode):
    """ Find legislators that represent the specified zipcode.
        Results are cached in the datastore for faster lookup.
        If more than one member of the House represents a zipcode,
        both legislators will be returned.

        zipcode: the 5-digit zipcode to search
    """

    # attempt to find cached legislators
    doc = g.db.legislatorsByZipcode.find_one({'zipcode': zipcode})

    if doc is None:

        # load from Sunlight Congress API if not cached locally
        results = sunlight.legislators.allForZip(zipcode)

        # create a copy of the Legislator object dict
        legislators = [r.__dict__.copy() for r in results]

        # sort the legislators by reverse title so Senators are listed
        # before members of the House
        legislators.sort(lambda x, y: -cmp(x['title'], y['title']))

        # move current title to short_title and update title with
        # a more readable version, create full name
        for l in legislators:
            l['short_title'] = l['title']
            l['title'] = TITLES.get(l['title'], 'Representative')
            l['fullname'] = "%s %s %s" % (l['title'], l['firstname'], l['lastname'])

        # save new zipcode results document
        g.db.legislatorsByZipcode.insert({
            'timestamp': g.now,
            'zipcode': zipcode,
            'legislators': legislators,
        })

    else:

        # get legislators from cache
        legislators = doc['legislators']

    return legislators

def resolve_entity_id(crp_id):
    """ Convert a CRP candidate ID into an IE entity ID.
        Cached locally for better performance.
    """

    doc = g.db.crpMapping.find_one({'crp_id': crp_id})

    if doc is None:
        entity_id = ie.entities.id_lookup("urn:crp:recipient", crp_id)[0]['id']
        g.db.crpMapping.insert({
            'crp_id': crp_id,
            'entity_id': entity_id,
        })
    else:
        entity_id = doc['entity_id']

    return entity_id

def top_contributors(legislator):
    entity_id = resolve_entity_id(legislator['crp_id'])
    contribs = ie.pol.contributors(entity_id, cycle='2012', limit=10)
    return contribs

def legislator_bio(legislator):
    entity_id = resolve_entity_id(legislator['crp_id'])
    metadata = ie.entities.metadata(entity_id)
    return metadata['metadata']['bio']

def committee_iter(committees):
    for comm in committees:
        yield comm.name
        if comm.subcommittees:
            for subcomm in comm.subcommittees:
                yield subcomm.name

def committees(legislator):
    comms = sunlight.committees.allForLegislator(g.legislator['bioguide_id'])
    names = " ".join("%s." % c for c in committee_iter(comms))
    return names

def recent_votes(legislator):

    VOTES = {
        'Yea': 'yes',
        'Nay': 'no',
    }

    url = "http://api.realtimecongress.org/api/v1/votes.json"

    voter_key = "voter_ids.%s" % legislator['bioguide_id']

    params = {
        'per_page': 5,
        'vote_type': 'passage',
        '%s__exists' % voter_key: True,
        'sections': "question,result,%s" % voter_key,
    }

    resp = requests.get(url, params=params, headers={'X-APIKEY': settings.SUNLIGHT_KEY})

    data = json.loads(resp.content)['votes']
    for vote in data:
        voted = vote['voter_ids'][legislator['bioguide_id']]
        vote['voted'] = VOTES.get(voted, voted)
        vote['question'] = vote['question'].split(':')[-1].strip()
        del vote['voter_ids']

    return data


import matplotlib.pyplot as plt
from scipy.stats import sem
import time
import math
from timeit import default_timer as timer
from typing import List, Tuple
from jsonpickle.unpickler import decode
from petrelic.bn import Bn
from stroll import Client, Server
import jsonpickle
from credential import AnonymousCredential, BlindSignature, DisclosureProof, IssueRequest, PublicKey, SecretKey, generate_key
from petrelic.multiplicative.pairing import G1, G1Element, G2, G2Element, GT

from issuer import Issuer
from user import User

## Utility functions for testing
def decode_data(data: bytes):
    return jsonpickle.decode(data.decode())

def get_keys(subscriptions: List[str]) -> Tuple[bytes, bytes]:
    sk_enc, pk_enc = Server.generate_ca(["username"] + subscriptions)
    return sk_enc, pk_enc

#### =============
#### SUCCESS CASES
#### =============

subscriptions = ["t1", "t2", "t3"]
queried_subscriptions = ["t1", "t2"]
username = "test"
message = b'30.00.00'
server, client = Server(), Client()

sk, pk = get_keys(subscriptions)
packets = [[], [], [], []]

## Test that the keys were generated correctly by the server
def key_generation():
    sk_enc, pk_enc = get_keys(subscriptions)
    sk: SecretKey = decode_data(sk_enc)
    pk: PublicKey = decode_data(pk_enc)

    return sk, pk

## Test the successful generation of a credential step by step, as well as a stroll request
def credential_issuance():
    # Generate issue request and test it
    issue_request_enc, user_state = client.prepare_registration(pk, username, subscriptions)
    issue_request: IssueRequest = decode_data(issue_request_enc)

    # Process registration and test it server-side
    blind_signature_enc = server.process_registration(sk, pk, issue_request_enc, username, subscriptions)
    blind_signature: BlindSignature = decode_data(blind_signature_enc)

    # Obtain credential client side and test it
    credential_enc = client.process_registration_response(pk, blind_signature_enc, user_state)
    credential: AnonymousCredential = decode_data(credential_enc)

    packets[0] += [len(credential_enc)] + [len(blind_signature_enc) * 2] + [len(issue_request_enc) * 2]

    return credential_enc

def credential_showing(credential_enc):
    # Create a secret stroll request (disclosure proof) and test it
    stroll_request_enc = client.sign_request(pk, credential_enc, message, queried_subscriptions)
    stroll_request: DisclosureProof = decode_data(stroll_request_enc)

    packets[1] += [len(credential_enc) * 2] + [len(stroll_request_enc) * 2]

    return stroll_request_enc

def credential_verification(stroll_request_enc):
  server.check_request_signature(pk, message, queried_subscriptions, stroll_request_enc)

  packets[2].append(len(stroll_request_enc))

## Test the successful generation of a credential step by step, as well as a stroll request
def test_successful_request():
    subscriptions = ["t1", "t2", "t3"]
    queried_subscriptions = ["t1", "t2"]
    username = "test"
    message = b'30.00.00'
    sk, pk = get_keys(subscriptions)
    server, client = Server(), Client()

    # Generate issue request and test it
    issue_request_enc, user_state = client.prepare_registration(pk, username, subscriptions)
    issue_request: IssueRequest = decode_data(issue_request_enc)

    # Process registration and test it server-side
    blind_signature_enc = server.process_registration(sk, pk, issue_request_enc, username, subscriptions)
    blind_signature: BlindSignature = decode_data(blind_signature_enc)

    # Obtain credential client side and test it
    credential_enc = client.process_registration_response(pk, blind_signature_enc, user_state)
    credential: AnonymousCredential = decode_data(credential_enc)

    # Create a secret stroll request (disclosure proof) and test it
    stroll_request_enc = client.sign_request(pk, credential_enc, message, queried_subscriptions)
    stroll_request: DisclosureProof = decode_data(stroll_request_enc)

    packets[3] += [(len(issue_request_enc) * 2 + len(blind_signature_enc) * 2) * 1.25] + [(len(credential_enc) * 2 + len(stroll_request_enc) * 2) * 1.25]

    # Test that the request is valid
    assert server.check_request_signature(pk, message, queried_subscriptions, stroll_request_enc)

### Perf eval
num_iter = 5000

res = [[], [], [], [], []]
fun = [key_generation, credential_issuance, credential_showing, credential_verification, test_successful_request]
for i in range(num_iter):
  cred = None
  issuance = None
  for j in range(0, 5):
    start = timer()
    if j == 1:
      cred = fun[j]()
    elif j == 2:
      issuance = fun[j](cred)
    elif j == 3:
      fun[j](issuance)
    else:
      fun[j]()

    end = timer()
    res[j].append(end - start)

# time
stems = list(map(lambda x: sem(x) * 150, res))
means = list(map(lambda x: sum(x) / len(x), res))

# com
stems_com = list(map(lambda x : sem(x)* 10, packets))
means_com = list(map(lambda x: sum(x) / len(x), packets))

plt.errorbar(["Key gen.", "Cred. issuance", "Cred. showing", "Req. verification", "Full request"], means, stems, linestyle='None', marker='s')
plt.ylabel("Execution Time (in ms)")
plt.savefig("exec_time.png")
plt.show()

plt.errorbar(["Credential issuance", "Credential showing", "Request verification", "Full request"], means_com, stems_com, linestyle='None', marker='s')
plt.ylabel("Packets Exchanged (bytes length)")
plt.savefig("packets.png")
plt.show()


#! /usr/bin/env python

import os
import shutil
import shlex
import argparse
import json
import inspect

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from multiprocessing import Pool
from os.path import abspath, join, isfile
from subprocess import Popen, PIPE
from itertools import product
from nibabel import streamlines
from nibabel.streamlines.array_sequence import ArraySequence
from dipy.tracking.streamline import length, set_number_of_points

def run(cmd, live_verbose=False):
    print('\n' + cmd)
    p = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE)
    output, error = p.communicate()
    if output:
        print(output.decode('latin_1'))
    if error:
        print(error.decode('latin_1'))

def assert_dir(dir_path):
    full_path = abspath(dir_path)
    if not os.path.isdir(full_path):
        print('Creating %s' % full_path)
        os.makedirs(full_path)

def remake_dir(dir_path):
    full_path = abspath(dir_path)
    if os.path.isdir(full_path):
        shutil.rmtree(full_path)
    os.makedirs(full_path)

def move(src, dest):
    print('Moving %s to %s' % (src, dest))
    shutil.move(src, dest)

def copy(src, dest):
    print('Copying %s to %s' % (src, dest))
    shutil.copyfile(src, dest)

def mrconvert(src, dest):
    cmd = 'mrconvert %s %s' % (src, dest)
    run(cmd)

def proc(subject, mrtrix_dir, n_threads=0):

    print('Performing processing of DWI data')

    assert_dir(mrtrix_dir)

    fa = join(mrtrix_dir, subject, 'fa.mif')
    if not isfile(fa):

        dwi = join(mrtrix_dir, subject, 'dwi_raw.nii.gz')
        bvecs = join(mrtrix_dir, subject, 'bvecs')
        bvals = join(mrtrix_dir, subject, 'bvals')
        json_file = join(mrtrix_dir, subject, 'info.json')

        # Convert data to mrtrix format
        raw = join(mrtrix_dir, subject, 'dwi_raw.mif')
        if not isfile(raw):
            cmd = 'mrconvert -fslgrad %s %s -json_import %s %s %s -force' % (
                    bvecs, bvals, json_file, dwi, raw)
            run(cmd)

        # Perform eddy correction
        prep = join(mrtrix_dir, subject, 'dwi_preproc.mif')
        if not isfile(prep):
            with open(json_file, 'r') as f:
                json_content = json.load(f)
            pe_dir = json_content['PhaseEncodingDirection']
            cmd = 'dwifslpreproc %s %s -rpe_none -pe_dir %s ' \
                  '-nthreads %i -force' % (
                   raw, prep, pe_dir, n_threads)
            run(cmd)

        # Create mask
        mask = join(mrtrix_dir, subject, 'mask.mif')
        if not isfile(mask):
            cmd = 'dwi2mask %s %s -force' % (prep, mask)
            run(cmd)

        # Compute tensor
        tensor = join(mrtrix_dir, subject, 'tensor.mif')
        if not isfile(tensor):
            b0 = join(mrtrix_dir, subject, 'b0.mif')
            mask = join(mrtrix_dir, subject, 'mask.mif')
            dwi = join(mrtrix_dir, subject, 'dwi_preproc.mif')
            cmd = 'dwi2tensor -b0 %s -mask %s %s %s -force' % (
                    b0, mask, dwi, tensor)
            run(cmd)

        # Compute metrics
        vector = join(mrtrix_dir, subject, 'vector.mif')
        adc = join(mrtrix_dir, subject, 'adc.mif')
        fa = join(mrtrix_dir, subject, 'fa.mif')
        ad = join(mrtrix_dir, subject, 'ad.mif')
        rd = join(mrtrix_dir, subject, 'rd.mif')

        if np.any([not isfile(f) for f in [vector, adc, fa, ad, rd]]):
            cmd = 'tensor2metric -vector %s -adc %s -fa %s -ad %s -rd %s \
                   -mask %s %s -force' % (
                   vector, adc, fa, ad, rd, mask, tensor)
            run(cmd)

    else:
        print('Subejct %s already processed. Skipping.' % subject)

def transform(subject, mrtrix_dir, template_dir):

    print('Computing transform from diffusion to template space')

    transform_dir = join(mrtrix_dir, subject, 'transform')
    assert_dir(transform_dir)

    fa_template = join(template_dir, 'fa_template_0.5mm.mif')
    template_mask = join(template_dir, 'template_mask_0.5mm.mif')
    warp1 = join(transform_dir, 'warp1.mif')

    if not os.path.isfile(warp1):
        warp2 = join(transform_dir, 'warp2.mif')
        fa = join(mrtrix_dir, subject, 'fa.mif')
        mask = join(mrtrix_dir, subject, 'mask.mif')
        trans = join(transform_dir, 'fa_template.mif')
        cmd = 'mrregister %s %s \
                -nl_warp %s %s \
                -mask1 %s -mask2 %s \
                -transformed %s \
                -type rigid_affine_nonlinear \
                -rigid_scale 0.25,0.5,0.8,1.0 \
                -affine_scale 0.7,0.8,1.0,1.0 \
                -nl_scale 0.5,0.75,1.0,1.0,1.0 \
                -nl_niter 5,5,5,5,5 \
                -datatype float32 -noreorientation -force' % (
                fa, fa_template, warp1, warp2, mask, template_mask, trans)
        run(cmd)
    else:
        print('Subject %s already processed. Skipping.' % subject)

def tracks(subject, mrtrix_dir, template_dir):

    print('Extracting MCP tracks')

    mcp_dir = join(template_dir, 'mcp')
    transform_dir = join(mrtrix_dir, subject, 'transform')
    tracks_dir = join(mrtrix_dir, subject, 'tracks')
    assert_dir(tracks_dir)

    # Create whole brain tractography
    mask = join(mrtrix_dir, subject, 'mask.mif')
    vector = join(mrtrix_dir, subject, 'vector.mif')
    tck = join(tracks_dir, 'tracks.tck')
    if not isfile(tck):
        cmd = 'tckgen -algorithm FACT -seed_image %s -mask %s -angle 30 \
                -select 100000 -minlength 20 %s %s -force' % (
                mask, mask, vector, tck)
        run(cmd)

    # Transfer tracks to template space
    warp = join(transform_dir, 'warp2.mif')
    tck_in = join(tracks_dir, 'tracks.tck')
    tck_temp = join(tracks_dir, 'tracks.template.tck')
    if not isfile(tck_temp):
        cmd = 'tcktransform %s %s %s -force' % (tck_in, warp, tck_temp)
        run(cmd)

    # Select tracks
    for hemi in ['L', 'R']:
        tck_out = join(tracks_dir, 'MCP_%s.template.tck' % hemi)
        roi = join(mcp_dir, 'MCP_%s.nii.gz' % hemi)
        excl = join(mcp_dir, 'MCP_%s_exclude.nii.gz' % hemi)
        cmd = 'tckedit -minlength 15 -include %s -exclude %s %s %s -force' % (
                    roi, excl, tck_temp, tck_out)
        run(cmd)

    os.remove(tck_temp)

def trim_track(stl, target_len='floor', pts_dist=None, n_pts=None):

    seg_len = np.linalg.norm(np.diff(stl, axis=0), axis=1)
    seg_cumlen = np.cumsum(seg_len)

    if target_len == 'floor':
        lim = np.floor(seg_cumlen[-1])
    else:
        lim = target_len

    len_thr = [x < lim for x in seg_cumlen]
    if not np.any(len_thr):
        current = np.linalg.norm(stl[1, :]-stl[0])
        target = lim
        stretch = target/current
        stl_trim = np.vstack((stl[0, :].copy(),
                             stl[0, :] + (stl[1, :] - stl[0, :])*stretch))
    else:
        i = np.where(len_thr)[0][-1] + 1
        current = np.linalg.norm(stl[i+1, :]-stl[i])
        target = lim - seg_cumlen[i-1]
        stretch = target/current
        stl_trim = np.vstack((stl[:i+1, :].copy(),
                             stl[i, :] + (stl[i+1, :] - stl[i, :])*stretch))

    if pts_dist is not None and int(lim/pts_dist) + 1 >= 2:
        stl_trim = set_number_of_points(stl_trim, int(lim/pts_dist) + 1)
    else:
        raise ValueError('Steamline is shorter than requested pts_dist.')

    if n_pts is not None:
        stl_trim = set_number_of_points(stl_trim, n_pts)

    return stl_trim

def tcktrim(fin, fout, target_len='floor', pts_dist=None, n_pts=None):

    tck_in = streamlines.load(fin)
    stl = tck_in.streamlines.copy()
    stl_trim = []
    for stl_ in stl:
        stl_trim.append(trim_track(stl_, target_len=target_len,
                        pts_dist=pts_dist, n_pts=n_pts))
    tck_out = streamlines.tck.TckFile(streamlines.Tractogram(
                    ArraySequence(stl_trim),
                    affine_to_rasmm=tck_in.tractogram.affine_to_rasmm))
    tck_out.save(fout)

def filter(subject, mrtrix_dir, template_dir):

    print('Filtering tracks prior to tractometry')

    mcp_dir = join(template_dir, 'mcp')
    tracks_dir = join(mrtrix_dir, subject, 'tracks')

    for hemi in ['L', 'R']:

        # Limit anterior part of tracks to selection ROI
        tck_out = join(tracks_dir, 'MCP_%s.template.ant_lim.tck' % hemi)
        if not isfile(tck_out):
            tck_in = join(tracks_dir, 'MCP_%s.template.tck' % hemi)
            mask = join(mcp_dir, 'MCP_%s_ant_lim.nii.gz' % hemi)
            cmd = 'tckedit -force -mask %s %s %s -force' % (
                        mask, tck_in, tck_out)
            run(cmd)

        tck_out = join(tracks_dir, 'MCP_%s.template.filter.tck' % hemi)
        if not isfile(tck_out):
            fname = join(tracks_dir, 'MCP_%s.template.ant_lim.tck' % hemi)
            tck_in = streamlines.load(fname)
            stl = tck_in.streamlines.copy()

            # Discard everything below 15mm and above 45mm
            stl_len = length(stl).reshape([-1, 1])
            ind = [x > 15. and x < 45. for x in stl_len]
            ind = [x[0] for x in ind]
            stl_thr = stl[ind].copy()
            stl_len = length(stl_thr).reshape([-1, 1])

            plt.hist(stl_len, bins=100, density=True)
            plt.title(subject)
            plt.xlabel('Length (mm)')
            plt.ylabel('Count')
            fname = join(tracks_dir, 'MCP_%s.template.histo.png' % hemi)
            plt.savefig(fname, format='png')
            plt.clf()

            # Match starts and ends

            # Get all start and stop num_points
            starts = np.vstack([stl_[0, :] for stl_ in stl_thr])
            stops = np.vstack([stl_[-1, :] for stl_ in stl_thr])

            # Reverse streamline if stop is more anterior than start
            reverse = [start < stop
                       for start, stop in zip(starts[:, 1], stops[:, 1])]
            stl_flip = list(stl_thr.copy())
            for ns in np.where(reverse)[0]:
                stl_flip[ns] = stl_flip[ns][::-1, :]

            # Trim streamlines to 1mm segments
            stl_len = length(stl_flip).reshape([-1, 1])
            stl_trim = []
            for stl_ in stl_flip:
                stl_trim.append(trim_track(stl_, target_len='floor', pts_dist=1.))

            # Visualize mean start and stop of streamlines for QA
            pts = np.vstack(stl_trim)
            starts = np.vstack([stl_[0, :] for stl_ in stl_trim])
            stops = np.vstack([stl_[-1, :] for stl_ in stl_trim])
            plt.plot(pts[:, 0], pts[:, 1], 'b.')
            plt.plot(starts[:, 0], starts[:, 1], 'g.')
            plt.plot(stops[:, 0], stops[:, 1], 'r.')
            plt.title(subject)
            plt.tight_layout()
            fname = join(tracks_dir, 'MCP_%s.filter.png' % hemi)
            plt.savefig(fname, format='png')
            plt.clf()

            # Save filtered tracks
            stl_final = ArraySequence(stl_trim)
            tck_filter = streamlines.tck.TckFile(
                            streamlines.Tractogram(stl_final,
                            affine_to_rasmm=tck_in.tractogram.affine_to_rasmm)
                      )
            tck_filter.save(tck_out)

        # Transfer tracks back to diffusion space
        tck_out = join(tracks_dir, 'MCP_%s.filter.tck' % hemi)
        if not isfile(tck_out):
            warp = join(mrtrix_dir, subject, 'transform', 'warp1.mif')
            tck_in = join(tracks_dir, 'MCP_%s.template.filter.tck' % hemi)
            cmd = 'tcktransform %s %s %s -force' % (tck_in, warp, tck_out)
            run(cmd)

def sample_tractogram_data(subject, mrtrix_dir):

    metrics = ['fa', 'adc', 'ad', 'rd']

    tracks_dir = join(mrtrix_dir, subject, 'tracks')
    metrics_dir = join(mrtrix_dir, subject, 'metrics')
    assert_dir(metrics_dir)

    for hemi in ['L', 'R']:

        # Sample metrics
        for metric in metrics:
            tck = join(tracks_dir, 'MCP_%s.filter.tck' % hemi)
            data = join(mrtrix_dir, subject, '%s.mif' % metric)
            outfile = join(metrics_dir, 'MCP_%s_%s.dat' %
                           (hemi, metric))

            cmd = 'tcksample %s %s %s -force' % (tck, data, outfile)
            run(cmd)

def extend_streamline_data(data, target_len=45):
    data_ext = []
    for nd, data_ in enumerate(data):
        if len(data_) < target_len:
            data_ext.append(np.hstack((data_,
                            [np.nan]*(target_len-len(data_)))))
        else:
            data_ext.append(data_[:target_len])
    return np.vstack(data_ext)

def export_tractogram_data(subjects, mrtrix_dir, stats_dir, target_len=30):

    tract_dir = join(stats_dir, 'tractogram_data')
    assert_dir(tract_dir)

    metrics = ['fa', 'adc', 'ad', 'rd']
    hemilist = ['L', 'R']

    length = range(0, target_len + 1)
    df = pd.DataFrame(index=length)
    df.index.name = 'length'

    # Extract metrics
    for subject in subjects:

        print('Processing %s' % subject)

        for metric in metrics:

            df_ = pd.DataFrame(index=length)
            df_.index.name = 'length'

            for hemi in hemilist:

                metrics_dir = join(mrtrix_dir, subject, 'metrics')

                fname = join(metrics_dir, 'MCP_%s_%s.dat' % (hemi, metric))
                with open(fname, 'r') as f:
                    lines = f.readlines()
                data = np.array([[float(x) for x in line.strip().split()]
                                for line in lines[1:]])
                data = extend_streamline_data(data, target_len + 1)

                if metric != 'fa':
                    data = data * 1000

                # All tracts
                df_ = pd.DataFrame(data.T)
                fname = join(tract_dir, '%s.%s.%s.csv' % (
                             subject, hemi, metric))
                df_.to_csv(fname)

                # Median data
                col = '%s.%s.%s' % (subject, hemi, metric)
                df.loc[:, col] = np.nanmedian(data, axis=0)

    # Save data
    fname = join(tract_dir, 'median.tracks.csv')
    df.to_csv(fname)

def tractogram_trimmed_metrics(subjects, mrtrix_dir, stats_dir, target_len=5):

    # Trim streamlines to length of interest
    print('Trimming steamlines')

    hemilist = ['L', 'R']
    for subject in subjects:
        # subject = 'vco1014.test'

        tracks_dir = join(mrtrix_dir, subject, 'tracks')

        for hemi in hemilist:
            tck_final = join(tracks_dir, 'MCP_%s.filter.%imm.tck' % (
                            hemi, target_len))
            if not isfile(tck_final):
                tck_in = join(tracks_dir, 'MCP_%s.template.filter.tck' % hemi)
                tck_out = join(tracks_dir, 'MCP_%s.template.filter.%imm.tck' % (
                               hemi, target_len))
                tcktrim(tck_in, tck_out, target_len=target_len, pts_dist=1.)
                # Transfer tracks back to MNI space
                warp = join(mrtrix_dir, subject, 'transform', 'warp1.mif')
                cmd = 'tcktransform %s %s %s -force' % (
                        tck_out, warp, tck_final)
                run(cmd)

    print('Extracting metrics')

    metrics = ['fa', 'adc', 'ad', 'rd']
    cols = ['MCP.%s.%s' % (hemi, metric)
            for hemi, metric in product(hemilist, metrics)]
    df = pd.DataFrame(index=subjects, columns=cols)
    df.index.name = 'subjects'

    for metric in metrics:
        for hemi in hemilist:
            for subject in subjects:

                tracks_dir = join(mrtrix_dir, subject, 'tracks')
                metrics_dir = join(mrtrix_dir, subject, 'metrics')

                tck = join(tracks_dir, 'MCP_%s.filter.%imm.tck' % (
                        hemi, target_len))
                data = join(mrtrix_dir, subject, '%s.mif' % metric)
                outfile = join(metrics_dir, 'MCP_%s_%s.%imm.dat' %
                               (hemi, metric, target_len))
                if not isfile(outfile):
                    cmd = 'tcksample %s %s %s -force' % (tck, data, outfile)
                    run(cmd)

                with open(outfile, 'r') as f:
                    lines = f.readlines()
                data = np.array([[float(x) for x in line.strip().split()]
                                 for line in lines[1:]])
                if metric != 'fa':
                    data = data * 1000

                col = 'MCP.%s.%s' % (hemi, metric)
                df.loc[subject, col] = np.mean(np.median(
                                               np.vstack(data), axis=0))

    assert_dir(stats_dir)
    fname = join(stats_dir, 'tracts.mean.%imm.csv' % target_len)
    df.to_csv(fname)

if __name__ == '__main__':

    main_dir = abspath(join(inspect.getfile(inspect.currentframe()),
                            os.pardir))

    parser = argparse.ArgumentParser(description="MCP tractograpy using MRtrix3")
    mutex = parser.add_mutually_exclusive_group()
    mutex.add_argument("-s", "--subject", type=str, default=None,
                       help="Subjects to be processed")
    mutex.add_argument("-sl", "--subjects_list", type=str, default=None,
                       help="List of all subject to be used for training.")
    parser.add_argument("-i", "--import_dicoms", action="store_true",
                        help="Import DWI data from DICOMs.")
    parser.add_argument("-p", "--proc", action="store_true",
                        help="Perform processing of DWI data.")
    parser.add_argument("-tr", "--transform", action="store_true",
                        help="Compute transformation from MNI to diffusion space")
    parser.add_argument("-t", "--tracks", action="store_true",
                        help="Extract MCP tracts from whole-brain \
                              tractography.")
    parser.add_argument("-f", "--filter", action="store_true",
                        help="Filter tracks to perform tractogram analysis.")
    parser.add_argument("-std", "--sample_tractogram_data", action="store_true",
                        help="Extract data for tractogram along MCP tracks.")
    parser.add_argument("-etd", "--export_tractogram_data", action="store_true",
                        help="Export median tractogram data to CSV.")
    parser.add_argument("-ttm", "--tractogram_trimmed_metrics",
                        action="store_true",
                        help="Extract mean of data along tracts trimmed at a given length.")
    parser.add_argument("-tl", "--target_length", type=float, default=10,
                        help="Target length for trimming.")
    parser.add_argument("-n_threads", "--n_threads", type=int, default=0,
                        help="Number of thread to be used by dwipreproc.")
    parser.add_argument("-n_jobs", "--n_jobs", type=int, default=1,
                        help="Number of parallel jobs. Default 1.")
    args = parser.parse_args()

    # Post process arguments
    if args.subject:
        subjects = [args.subject]

    if args.subjects_list:
        with open(args.subjects_list, 'r') as f:
            subjects = [subject.strip() for subject in f.readlines()]

    # Define main directories
    dicoms_dir = join(main_dir, 'dicoms')
    mrtrix_dir = join(main_dir, 'mrtrix')
    template_dir = join(main_dir, 'population_template')
    stats_dir = join(main_dir, 'stats')

    # Run the commands

    pool = Pool(processes=args.n_jobs)

    if args.proc:
        params = list(product(subjects, [mrtrix_dir], [args.n_threads]))
        r = pool.starmap_async(proc, params)
        r.wait()

    if args.transform:
        params = list(product(subjects, [mrtrix_dir], [template_dir]))
        r = pool.starmap_async(transform, params)
        r.wait()

    if args.tracks:
        params = list(product(subjects, [mrtrix_dir], [template_dir]))
        r = pool.starmap_async(tracks, params)
        r.wait()

    if args.filter:
        params = list(product(subjects, [mrtrix_dir], [template_dir]))
        r = pool.starmap_async(filter, params)
        r.wait()

    if args.sample_tractogram_data:
        params = list(product(subjects, [mrtrix_dir]))
        r = pool.starmap_async(sample_tractogram_data, params)
        r.wait()

    pool.close()
    pool.join()

    if args.export_tractogram_data:
        export_tractogram_data(subjects, mrtrix_dir, stats_dir)

    if args.tractogram_trimmed_metrics:
        tractogram_trimmed_metrics(
                        subjects,
                        mrtrix_dir,
                        stats_dir,
                        args.target_length
        )


import os
import vtk, qt, ctk, slicer
import logging
import numpy as np
from DICOMLib import DICOMUtils

#########################################################
#
#

comment = """

  OrthovoltageDoseEngineUtil contains utility functions
  for calling ctcreate from Slicer. Ctcreate converts an
  image file (ie DICOM CT) to an .egsphant file, which
  acts as input to DOSXYZnrc for dosimetry planning.

"""

#-----------------------------------------------------------------------------
def generateSlicenamesTextfile(ctDicomSeriesUID, slicenamesFilename,
  outputFolder):
  """ Generate slicenames.txt file, with list of ct dicom slices, in increasing slice order (IS direction)
  """
  filePaths = slicer.dicomDatabase.filesForSeries(ctDicomSeriesUID)
  if len(filePaths) == 0:
    logging.error('Failed to find files in DICOM database for UID ' + str(ctDicomSeriesUID))
    return False

  unsortedFileList = slicer.dicomDatabase.filesForSeries(ctDicomSeriesUID)
  sortedFileList, distances, warnings = DICOMUtils.getSortedImageFiles(unsortedFileList)

  outFile = open(os.path.join(outputFolder, slicenamesFilename), "wb")
  counter = 1
  numDicomFiles = len(sortedFileList)
  for sliceFileName in   sortedFileList:
    outFile.write(sliceFileName)
    if counter != numDicomFiles:
      outFile.write("\n")
    counter += 1
  outFile.close()
  return True

#-----------------------------------------------------------------------------
def generateCtcreateInputFile(slicenamesFilename, imageROI, voxelThickness,
  outputFolder):
  """ Generate ctcreate.inp file, which is used to execute ctcreate
      Input records are described in the DOSXYZnrc Users Manual.
  """
  outFile = open(os.path.join(outputFolder, "ctcreate.inp"), "wb")

  #CT Record 1 ctformat
  outFile.write("DICOM\n")

  #CT Record 2 CTFilename
  outFile.write(os.path.join(outputFolder, slicenamesFilename) + "\n")

  #CT Record 3 lower and upper boundaries (cm) to be considered for
  #the dosxyznrc phantom
  outFile.write(", ".join(map(str, imageROI)) + "\n")

  #CT Record 4 x, y, z voxel dimensions/thicknesses (cm) to be used
  #for the dosxyznrc phantom
  outFile.write(", ".join(map(str, voxelThickness)) + "\n")

  #CT Record 5 num_material, material_ct_lower_bound
  outFile.write("4, -1024\n")

  """ Record 6 defines the material name, followed by the ramp parameters
      for each material. The ramp parameters are: material ct upper bound,
      material density lower bound, material density upper bound.

      The CT ramp is used to determine the medium and density in each voxel
      of the CT data. The material names must correspond to materials in the
      PEGS4 data file being used in the DOSXYZnrc simulation.

      The values used here are the default values specified for DOSXYZnrc.
      It is possible to create a custom CT ramp based on the imager and the
      data acquisition method, but since Slicer users will likely not have
      a custom CT ramp to accompany their input CT, material information
      is made non-configurable here and default values are used.

      For more information on materials and ramps, please see the DOSXYZnrc
      Users Manual.
  """
  #CT Record 6 information about material (for i=1 to num_material)
  outFile.write("AIR521ICRU\n")
  outFile.write("-974, 0.001, 0.044\n")
  outFile.write("LUNG521ICRU\n")
  outFile.write("-724, 0.044, 0.302\n")
  outFile.write("ICRUTISSUE521ICRU\n")
  outFile.write("101, 0.302, 1.101\n")
  outFile.write("ICRPBONE521ICRU\n")
  outFile.write("1976, 1.101, 2.088")
  outFile.close()

#-----------------------------------------------------------------------------
def generateCtcreateInput(volumeNode, ctDicomSeriesUID, outputFolder, imageROIMm=None,
  voxelThicknessMm=None):
  """ Generate all files needed as input to ctcreate

      NOTE: need to supply outputFolder path with 2 slashes (ie "C:\\d\\outputFolder")
            otherwise os.path.join may misbehave
  """
  slicenamesFilename = "slicenames.txt"

  if imageROIMm is None and volumeNode is None:
    logging.error('No information provided for desired image ROI in ctcreate \
      phantom. Please provide a volume node, or imageROIMm parameter.')
    return False
  #If no ROI list provided, get ROI from volume node
  elif imageROIMm is None:
    imageROIMm = [0] * 6
    volumeNode.GetBounds(imageROIMm)

  if voxelThicknessMm is None and volumeNode is None:
    logging.error('No information provided for desired voxel thickness in ctcreate \
      phantom. Please provide a volume node, or volumeThicknessMm parameter.')
    return False
  #If no voxel thickness list provided, get voxel thickness from volume node
  elif voxelThicknessMm is None:
    voxelThicknessMm = volumeNode.GetSpacing()

  #Convert ROI and voxelThickness from mm to cm
  imageROICm = [dimension/10 for dimension in imageROIMm]
  voxelThicknessCm = [dimension/10 for dimension in voxelThicknessMm]

  generateSlicenamesTextfile(ctDicomSeriesUID, slicenamesFilename,
    outputFolder)
  generateCtcreateInputFile(slicenamesFilename, imageROICm, voxelThicknessCm,
    outputFolder)
  return True

#-----------------------------------------------------------------------------
def callCtcreate(outputFolder, ctcreateInputFilename="ctcreate.inp"):
  """ Call ctcreate executable. Use this function after generating input for ctcreate
  """
  #If egsphant file exists, remove it
  outputCtcreatePhantomPath = os.path.join(outputFolder, "slicenames.txt.egsphant")
  if os.path.exists(outputCtcreatePhantomPath):
    logging.warning("Ctcreate phantom already exists in specifying directory. Overwriting it.")
    os.remove(outputCtcreatePhantomPath)

  #User must have ctcreate installed and in path
  os.system("ctcreate " + os.path.join(outputFolder, ctcreateInputFilename))


import json
from web3 import Web3
from web3.contract import ConciseContract

# web3.py instance
w3 = Web3(Web3.HTTPProvider("http://127.0.0.1:8545"))

with open('artifacts/OceanToken.spree.json', 'r') as infile:
    parce = json.loads(infile.read())
abi = parce['abi']
contract_address = parce['address']
token = w3.eth.contract(
    address=contract_address,
    abi=abi,
)
print('Address: {}'.format(
    contract_address
))

concise_token = ConciseContract(token)

print('Name: {}'.format(
    token.functions.name().call()
))

w3.personal.unlockAccount(w3.eth.accounts[1], 'secret')
tx_hash = token.functions.mint(w3.eth.accounts[1],100000000000000).transact({'from': w3.eth.accounts[1]})
tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)

w3.personal.unlockAccount(w3.eth.accounts[1], 'secret')
tx_hash = token.functions.mint(w3.eth.accounts[2],100000000000000).transact({'from': w3.eth.accounts[1]})
tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)

w3.personal.unlockAccount(w3.eth.accounts[1], 'secret')
tx_hash = token.functions.mint(w3.eth.accounts[3],100000000000000).transact({'from': w3.eth.accounts[1]})
tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)

print ("Balances: ")
print('Account 1: {}'.format(
    token.functions.balanceOf(w3.eth.accounts[1]).call()
))

print('Account 2: {}'.format(
    token.functions.balanceOf(w3.eth.accounts[2]).call()
))

print('Account 3: {}'.format(
    token.functions.balanceOf(w3.eth.accounts[3]).call()
))

with open('artifacts/DirectPurchase.spree.json', 'r') as infile:
    parce = json.loads(infile.read())
abi = parce['abi']
purchase_address = parce['address']
purchase_contract = w3.eth.contract(
    address=purchase_address,
    abi=abi,
)

concise_purchase = ConciseContract(purchase_contract)

w3.personal.unlockAccount(w3.eth.accounts[1], 'secret')
tx_hash = token.functions.approve(purchase_address, 100).transact({'from': w3.eth.accounts[1]})
tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)

w3.personal.unlockAccount(w3.eth.accounts[1], 'secret')
tx_hash = concise_purchase.sendTokenAndLog(w3.eth.accounts[3], 100, Web3.toBytes(50), Web3.toBytes(50), transact = {'from': w3.eth.accounts[1]})
tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash)

print('Transaction log raw: {}'.format(tx_receipt['logs'][2]['data']))
print('Transaction log indexed:')
event_filter = purchase_contract.events.TokenSent.createFilter(fromBlock=1, toBlock="latest", argument_filters={'_from':w3.eth.accounts[1], '_to':w3.eth.accounts[3], '_reference2':Web3.toBytes(50)})
event = event_filter.get_all_entries()[0].args

print(event._from)
print(event._to)
print(event._amount)
print(event._reference1)
print(event._reference2)

print ("Balances: ")
print('Account 1: {}'.format(
    token.functions.balanceOf(w3.eth.accounts[1]).call()
))

print('Account 2: {}'.format(
    token.functions.balanceOf(w3.eth.accounts[2]).call()
))

print('Account 3: {}'.format(
    token.functions.balanceOf(w3.eth.accounts[3]).call()
))


class Solution:
    def intersect(self, nums1, nums2):
        ans = []
        for num in nums1:
            if num not in ans:
                for _ in range(min(nums1.count(num), nums2.count(num))):
                    ans.append(num)

        return ans

class Solution2:
    def intersect(self, nums1, nums2):
        freq = {}
        ans = []
        for num in nums1:
            if num in freq.keys():
                freq[num][0] += 1
            else:
                freq[num] = [1, 0]
        for num in nums2:
            if num in freq.keys():
                freq[num][1] += 1
            else:
                freq[num] = [0, 1]

        for k, v in freq.items():
            ans.extend([k] * min(v))

        return ans

class Solution3:
    def intersect(self, nums1, nums2):
        nums1.sort()
        nums2.sort()
        ans = []
        i = j = 0

        while i < len(nums1) and j<len(nums2):
            if nums1[i] == nums2[j]:
                ans.append(nums1[i])
                i += 1
                j += 1
            else:
                if nums1[i] < nums2[j]:
                    while i < len(nums1) and nums1[i] < nums2[j]:
                        i += 1
                else:
                    while j < len(nums2) and nums2[j] < nums1[i]:
                        j += 1
        return ans


#chunking similar words
import nltk
from nltk.corpus import state_union
from nltk.tokenize import PunktSentenceTokenizer

train_text = state_union.raw("2005-GWBush.txt") #training sample
sample_text = state_union.raw("2006-GWBush.txt") #testing sample

custom_sent_tokenizer = PunktSentenceTokenizer(train_text) #trains any body of text, passed training sample through Punkt function

tokenized = custom_sent_tokenizer.tokenize(sample_text) #pass testing sample through custom tokenizer that was trained

def content_process(): #this function will tag all parts of speech per sentence
    try:
        for word in tokenized:
            words = nltk.word_tokenize(word) #tokenizes file by word
            tagged = nltk.pos_tag(words) #tags parts of speech

            chunkGram = r"""Chunk: {<.*>+}
                                }<VB.?|IN|DT>{""" #chunks by parts of speech in sentence. Arguments passed through opposite curly braces are the POS's that won't be chunked.

            chunkParser = nltk.RegexpParser(chunkGram) #creates a visualization of chunked data
            chunked = chunkParser.parse(tagged)

        print(chunked)
        for subtree in chunked.subtrees(filter= lambda t: t.label() == 'Chunk'):
            print(subtree)

        chunked.draw()

    except Exception as e:
        print(str(e))

print(content_process())
print(chunkGram)


import tensorflow as tf
from ssd.ssd_layers import Normalize
from ssd.ssd_layers import PriorBox
from ssd.ssd_layers import AtrousConvolution2D

class SSD(object):
    def __init__(self):
        super().__init__()
        #self.input_shape = input_shape
        #self.num_classes = num_classes

        self.fc6 = AtrousConvolution2D((3,3,512,1024), name="fc6")
        #self.conv4_3_norm = Normalize(20, name="conv4_3_norm")

    def __call__(self, input_tensor):

        #input_shape = self.input_shape
        #num_classes = self.num_classes

        # input image size
        #img_size = (input_shape[1], input_shape[0])
        # block1
        h_conv1_1 = tf.layers.conv2d(input_tensor, 64, [3, 3], padding="same",name="conv1_1", activation=tf.nn.relu)
        h_conv1_2 = tf.layers.conv2d(h_conv1_1, 64, [3, 3], padding="same",name="conv1_2", activation=tf.nn.relu)
        h_pool1 = tf.layers.max_pooling2d(h_conv1_2, 2, 2, name="pool1", padding="same")

        # block2
        h_conv2_1 = tf.layers.conv2d(h_pool1, 128, [3, 3], padding="same",name="conv2_1", activation=tf.nn.relu)
        h_conv2_2 = tf.layers.conv2d(h_conv2_1, 128, [3, 3], padding="same",name="conv2_2", activation=tf.nn.relu)
        h_pool2 = tf.layers.max_pooling2d(h_conv2_2, 2, 2, name="pool2", padding="same")

        # block3
        h_conv3_1 = tf.layers.conv2d(h_pool2, 256, [3, 3], padding="same",name="conv3_1", activation=tf.nn.relu)
        h_conv3_2 = tf.layers.conv2d(h_conv3_1, 256, [3, 3], padding="same",name="conv3_2", activation=tf.nn.relu)
        h_conv3_3 = tf.layers.conv2d(h_conv3_2, 256, [3, 3], padding="same",name="conv3_3", activation=tf.nn.relu)
        h_pool3 = tf.layers.max_pooling2d(h_conv3_3, 2, 2, name="pool3", padding="same")

        # block4
        h_conv4_1 = tf.layers.conv2d(h_pool3, 512, [3, 3], padding="same",name="conv4_1", activation=tf.nn.relu)
        h_conv4_2 = tf.layers.conv2d(h_conv4_1, 512, [3, 3], padding="same",name="conv4_2", activation=tf.nn.relu)
        h_conv4_3 = tf.layers.conv2d(h_conv4_2, 512, [3, 3], padding="same",name="conv4_3", activation=tf.nn.relu)
        h_pool4 = tf.layers.max_pooling2d(h_conv4_3, 2, 2, name="pool4", padding="same")

        # block5
        h_conv5_1 = tf.layers.conv2d(h_pool4, 512, [3, 3], padding="same",name="conv5_1", activation=tf.nn.relu)
        h_conv5_2 = tf.layers.conv2d(h_conv5_1, 512, [3, 3], padding="same",name="conv5_2", activation=tf.nn.relu)
        h_conv5_3 = tf.layers.conv2d(h_conv5_2, 512, [3, 3], padding="same",name="conv5_3", activation=tf.nn.relu)
        h_pool5 = tf.layers.max_pooling2d(h_conv5_3, 3, 1, name="pool5", padding="same")

        # FC6 (Full-Context?)
        #fc6 = AtrousConvolution2D((3,3,512,1024), name="fc6")
        fc6 = self.fc6
        h_fc6 = fc6(h_pool5)

        # FC7
        h_fc7 = tf.layers.conv2d(h_fc6, 512, [1, 1], padding="same",name="fc7", activation=tf.nn.relu)

        """
        # block6, subsampling
        h_conv6_1 = tf.layers.conv2d(h_fc7, 256, [1, 1], padding="same",name="conv6_1", activation=tf.nn.relu)
        h_conv6_2 = tf.layers.conv2d(h_conv6_1, 512, [3, 3], strides=(2, 2), padding="same",name="conv6_2", activation=tf.nn.relu)

        # block7, subsampling
        h_conv7_1 = tf.layers.conv2d(h_conv6_2, 128, [1, 1], padding="same",name="conv7_1", activation=tf.nn.relu)
        #h_conv7_1 = tf.keras.layers.ZeroPadding2D()(h_conv7_1)
        h_conv7_2 = tf.layers.conv2d(h_conv7_1, 256, [3, 3], strides=(2, 2), padding="same",name="conv7_2", activation=tf.nn.relu) # maybe "same" is also fine?

        # block8
        h_conv8_1 = tf.layers.conv2d(h_conv7_2, 128, [1, 1], padding="same",name="conv8_1", activation=tf.nn.relu)
        h_conv8_2 = tf.layers.conv2d(h_conv8_1, 256, [3, 3], strides=(2, 2), padding="same",name="conv8_2", activation=tf.nn.relu)

        # last pool(pool6), global average pool2d, return (1,1,channels)
        #print(tf.shape(h_conv8_2))
        #height, width = tf.to_int32(tf.shape(h_conv8_2)[1]), tf.to_int32(tf.shape(h_conv8_2)[2])
        #print(height)
        #print(width)
        #_, height, width, _ = tf.shape(h_conv8_2)
        #h_pool6 = tf.layers.average_pooling2d(h_conv8_2, [height, width], [height,width])
        h_pool6 = tf.reduce_mean(h_conv8_2, [1,2])
        """
        # return feature vectors
        return h_conv4_3, h_fc7#, h_conv6_2, h_conv7_2, h_conv8_2, h_pool6

class Detector(object):
    def __init__(self, input_shape, num_classes):

        self.input_shape = input_shape
        self.num_classes = num_classes
        #self.fc6 = AtrousConvolution2D((3,3,512,1024), name="fc6")
        self.conv4_3_norm = Normalize(20, name="conv4_3_norm")

    def __call__(self, feats, PRIORS=False):

        input_shape = self.input_shape
        num_classes = self.num_classes
        #h_conv4_3, h_fc7, h_conv6_2, h_conv7_2, h_conv8_2, h_pool6 = feats
        h_conv4_3, h_fc7 = feats

        img_size = (input_shape[1], input_shape[0])
        ###########################
        # prediction from conv4_3##
        ###########################
        h_conv4_3_norm = self.conv4_3_norm(h_conv4_3)
        num_priors = 3
        ## loc
        h_conv4_3_norm_mbox_loc = tf.layers.conv2d(h_conv4_3_norm, num_priors*4, [3, 3], padding="same",name="conv4_3_norm_mbox_loc", activation=None)
        ## flatten: y = tf.layers.Flatten()(x)
        h_conv4_3_norm_mbox_loc_flat = tf.layers.Flatten()(h_conv4_3_norm_mbox_loc)
        ## conf
        name = "conv4_3_norm_mbox_conf_{}".format(num_classes)
        h_conv4_3_norm_mbox_conf = tf.layers.conv2d(h_conv4_3_norm, num_priors*num_classes, [3, 3], padding="same", name=name, activation=None)

        ### flatten conf
        h_conv4_3_norm_mbox_conf_flat = tf.layers.Flatten()(h_conv4_3_norm_mbox_conf)
        ## priorbox
        priorbox = PriorBox(img_size, 30.0, aspect_ratios=[1/2., 2],
                        variances=[0.1, 0.1, 0.2, 0.2],
                        name='conv4_3_norm_mbox_priorbox')
        h_conv4_3_norm_mbox_priorbox = priorbox(h_conv4_3_norm)

        ########################
        # prediction from fc7###
        ########################
        num_priors = 6
        ## loc
        h_fc7_mbox_loc = tf.layers.conv2d(h_fc7, num_priors*4, [3, 3], padding="same", name="fc7_mbox_loc", activation=None)
        ## flatten: y = tf.layers.Flatten()(x)
        h_fc7_mbox_loc_flat = tf.layers.Flatten()(h_fc7_mbox_loc)
        ## conf
        name = "fc7_mbox_conf_{}".format(num_classes)
        h_fc7_mbox_conf = tf.layers.conv2d(h_fc7, num_priors*num_classes, [3, 3], padding="same", name=name, activation=None)
        ## flatten: y = tf.layers.Flatten()(x)
        h_fc7_mbox_conf_flat = tf.layers.Flatten()(h_fc7_mbox_conf)
        ## priorbox
        priorbox = PriorBox(img_size, 60.0, max_size=114.0, aspect_ratios=[1/2., 2, 3], variances=[0.1, 0.1, 0.2, 0.2], name="fc7_mbox_priorbox")
        h_fc7_mbox_priorbox = priorbox(h_fc7)
        """
        ##########################
        # prediction from conv6_2#
        ##########################
        num_priors = 6
        ## loc
        h_conv6_2_mbox_loc = tf.layers.conv2d(h_conv6_2, num_priors*4, [3, 3], padding="same", name="conv6_2_mbox_loc", activation=None)
        ### flatten
        h_conv6_2_mbox_loc_flat = tf.layers.Flatten()(h_conv6_2_mbox_loc)
        ## conf
        name = "conv6_2_mbox_conf_{}".format(num_classes)
        h_conv6_2_mbox_conf = tf.layers.conv2d(h_conv6_2, num_priors*num_classes, [3, 3], padding="same", name=name, activation=None)
        ### flatten
        h_conv6_2_mbox_conf_flat = tf.layers.Flatten()(h_conv6_2_mbox_conf)
        ## priorbox
        priorbox = PriorBox(img_size, 114.0, max_size=168.0, aspect_ratios=[1/2., 2, 3], variances=[0.1,0.1,0.2,0.2], name="conv6_2_mbox_priorbox")
        h_conv6_2_mbox_priorbox = priorbox(h_conv6_2)

        ##########################
        # prediction from conv7_2#
        ##########################
        num_priors = 6
        ## loc
        h_conv7_2_mbox_loc = tf.layers.conv2d(h_conv7_2, num_priors*4, [3, 3], padding="same", name="conv7_2_mbox_loc", activation=None)
        ### flatten: y = tf.layers.Flatten()(x)
        h_conv7_2_mbox_loc_flat = tf.layers.Flatten()(h_conv7_2_mbox_loc)
        ## conf
        name = "conv7_2_mbox_conf_{}".format(num_classes)
        h_conv7_2_mbox_conf = tf.layers.conv2d(h_conv7_2, num_priors*num_classes, [3, 3], padding="same", name=name, activation=None)
        ### flatten: y = tf.layers.Flatten()(x)
        h_conv7_2_mbox_conf_flat = tf.layers.Flatten()(h_conv7_2_mbox_conf)
        ## priorbox
        priorbox = PriorBox(img_size, 168.0, max_size=222.0, aspect_ratios=[2, 3], variances=[0.1, 0.1, 0.2, 0.2], name="conv7_2_mbox_priorbox")
        h_conv7_2_mbox_priorbox = priorbox(h_conv7_2)

        ##########################
        # prediction from conv8_2#
        ##########################
        num_priors = 6
        ## loc
        h_conv8_2_mbox_loc = tf.layers.conv2d(h_conv8_2, num_priors*4, [3, 3], padding="same", name="conv8_2_mbox_loc", activation=None)
        ## flatten
        h_conv8_2_mbox_loc_flat = tf.layers.Flatten()(h_conv8_2_mbox_loc)
        ## conf
        name = "conv8_2_mbox_conf_{}".format(num_classes)
        h_conv8_2_mbox_conf = tf.layers.conv2d(h_conv8_2, num_priors*num_classes, [3, 3], padding="same", name=name, activation=None)
        ## flatten
        h_conv8_2_mbox_conf_flat = tf.layers.Flatten()(h_conv8_2_mbox_conf)
        ## priorbox
        priorbox = PriorBox(img_size, 222.0, max_size=276.0, aspect_ratios=[2, 3], variances=[0.1,0.1,0.2,0.2], name="conv8_2_mbox_priorbox")
        h_conv8_2_mbox_priorbox = priorbox(h_conv8_2)

        #########################
        # prediction from pool6 #
        #########################
        num_priors = 6
        ## loc dense
        h_pool6_mbox_loc_flat = tf.layers.dense(h_pool6, num_priors*4, name="pool6_mbox_loc_flat", activation=None)
        ## conf
        name = "pool6_mbox_conf_flat_{}".format(num_classes)
        h_pool6_mbox_conf_flat = tf.layers.dense(h_pool6, num_priors*num_classes, name=name, activation=None)
        ## priorbox
        priorbox = PriorBox(img_size, 222.0, max_size=276.0, aspect_ratios=[2, 3], variances=[0.1,0.1,0.2,0.2], name="conv8_2_mbox_priorbox")
        ### reshape h_pool6
        target_shape = (1,1) + (h_pool6.get_shape().as_list()[-1],)
        h_pool6_reshaped = tf.keras.layers.Reshape(target_shape, name="pool6_reshaped")(h_pool6)
        h_pool6_mbox_priorbox = priorbox(h_pool6_reshaped)
        """
        ##########################
        # Gather all predictions #
        ##########################
        """
        h_mbox_loc = tf.concat( [h_conv4_3_norm_mbox_loc_flat,
                                 h_fc7_mbox_loc_flat,
                                 h_conv6_2_mbox_loc_flat,
                                 h_conv7_2_mbox_loc_flat,
                                 h_conv8_2_mbox_loc_flat,
                                 h_pool6_mbox_loc_flat], axis = 1, name = "mbox_loc")
        h_mbox_conf = tf.concat( [h_conv4_3_norm_mbox_conf_flat,
                                 h_fc7_mbox_conf_flat,
                                 h_conv6_2_mbox_conf_flat,
                                 h_conv7_2_mbox_conf_flat,
                                 h_conv8_2_mbox_conf_flat,
                                 h_pool6_mbox_conf_flat], axis = 1, name = "mbox_conf")
        h_mbox_priorbox = tf.concat( [h_conv4_3_norm_mbox_priorbox,
                                 h_fc7_mbox_priorbox,
                                 h_conv6_2_mbox_priorbox,
                                 h_conv7_2_mbox_priorbox,
                                 h_conv8_2_mbox_priorbox,
                                 h_pool6_mbox_priorbox], axis = 1, name = "mbox_priorbox")
        """
        h_mbox_loc = tf.concat( [h_conv4_3_norm_mbox_loc_flat,
                                 h_fc7_mbox_loc_flat], axis = 1, name = "mbox_loc")
        h_mbox_conf = tf.concat( [h_conv4_3_norm_mbox_conf_flat,
                                 h_fc7_mbox_conf_flat], axis = 1, name = "mbox_conf")
        h_mbox_priorbox = tf.concat( [h_conv4_3_norm_mbox_priorbox,
                                 h_fc7_mbox_priorbox], axis = 1, name = "mbox_priorbox")

        # count number of prior bboxs
        num_boxes = tf.shape(h_mbox_loc)[-1] // 4
        h_mbox_loc = tf.keras.layers.Reshape((num_boxes, 4), name="mbox_loc_final")(h_mbox_loc)
        h_mbox_conf = tf.keras.layers.Reshape((num_boxes, num_classes), name="mbox_conf_logits")(h_mbox_conf)
        h_mbox_conf = tf.nn.softmax(h_mbox_conf, name="mbox_conf_final")

        # final prediction
        predictions = tf.concat([h_mbox_loc, h_mbox_conf, h_mbox_priorbox], axis=2, name="predictions")

        if PRIORS:
            return h_mbox_priorbox
        else:
            return predictions

class UpSample(object):
    def __init__(self):
        pass

    def __call__(self, feats):
        h_conv4_3, h_fc7, h_conv6_2, h_conv7_2, h_conv8_2, h_pool6 = feats

        up_factors = []

        # upsampling
        ## fc7 up
        up_factor = h_conv4_3.get_shape().as_list()[1]//h_fc7.get_shape().as_list()[1]
        fc7_up = tf.keras.layers.UpSampling2D(up_factor, name="fc_up")
        h_fc7_up = fc7_up(h_fc7)
        up_factors.append(up_factor)
        ## conv6_2 up
        up_factor = h_conv4_3.get_shape().as_list()[1]//h_conv6_2.get_shape().as_list()[1]
        conv6_2_up = tf.keras.layers.UpSampling2D(up_factor, name="conv6_2_up")
        h_conv6_2_up = conv6_2_up(h_conv6_2)
        up_factors.append(up_factor)
        ## conv7_2 up
        up_factor = h_conv4_3.get_shape().as_list()[1]//h_conv7_2.get_shape().as_list()[1]
        conv7_2_up = tf.keras.layers.UpSampling2D(up_factor, name="conv7_2_up")
        h_conv7_2_up = conv6_2_up(h_conv6_2)
        up_factors.append(up_factor)
        ## conv8_2 up
        up_factor = h_conv4_3.get_shape().as_list()[1]//h_conv8_2.get_shape().as_list()[1]
        conv8_2_up = tf.keras.layers.UpSampling2D(up_factor, name="conv8_2_up")
        h_conv8_2_up = conv6_2_up(h_conv6_2)
        up_factors.append(up_factor)
        ## pool6 up
        h_pool6 = tf.expand_dims(h_pool6, 1)
        h_pool6 = tf.expand_dims(h_pool6, 1)

        up_factor = h_conv4_3.get_shape().as_list()[1]//h_pool6.get_shape().as_list()[1]
        pool6_up = tf.keras.layers.UpSampling2D(up_factor, name="pool6_up")
        h_pool6_up = pool6_up(h_pool6)
        up_factors.append(up_factor)

        up_feats = [h_conv4_3, h_fc7_up, h_conv6_2_up, h_conv7_2_up, h_conv8_2_up, h_pool6_up]

        # concatenate together
        up_feats_concat = tf.concat(up_feats, axis=3, name="up_feats_concat")

        return up_feats_concat, up_factors

## downsample for final detection
class DownSample(object):
    def __init__(self):
        pass

    def test(self):
        print("hhh")

    def __call__(self, feats_concat, up_factors):
        h_conv4_3 = feats_concat[:,:,:,:512] # 512
        h_fc7_up = feats_concat[:,:,:,512:512+1024] # 1024
        h_conv6_2_up = feats_concat[:,:,:,512+1024:2048] # 512
        h_conv7_2_up = feats_concat[:,:,:,2048:2048+256] # 256
        h_conv8_2_up = feats_concat[:,:,:,2048+256:2048+256+256] # 256
        h_pool6_up = feats_concat[:,:,:,2048+512:2048+512+256] # 256

        # downsample by pooling
        fc7_down = tf.keras.layers.AveragePooling2D(up_factors[0], padding="same", name="fc7_down")
        h_fc7 = fc7_down(h_fc7_up)

        conv6_2_down = tf.keras.layers.AveragePooling2D(up_factors[1], padding="same", name="conv6_2_down")
        h_conv6_2 = conv6_2_down(h_conv6_2_up)

        conv7_2_down = tf.keras.layers.AveragePooling2D(up_factors[2], padding="same", name="conv7_2_down")
        h_conv7_2 = conv7_2_down(h_conv7_2_up)

        conv8_2_down = tf.keras.layers.AveragePooling2D(up_factors[3], padding="same", name="conv8_2_down")
        h_conv8_2 = conv8_2_down(h_conv8_2_up)

        pool6_down = tf.keras.layers.AveragePooling2D(up_factors[4], padding="same", name="pool6_down")
        h_pool6 = pool6_down(h_pool6_up)
        h_pool6 = tf.squeeze(h_pool6, [1,2], name="pool6_squeeze")

        return h_conv4_3, h_fc7, h_conv6_2, h_conv7_2, h_conv8_2, h_pool6


from rest_framework.response import Response
from rest_framework.views import APIView
from teacherapp.models import Teacher
from teacherapp.serializers import TeacherDeSerializer, TeacherSerializer

class TeacherAPIView(APIView):

    def get(self,request,*args,**kwargs):
        tea_id = kwargs.get('id')

        if tea_id:
            tea_obj = Teacher.objects.get(pk=tea_id)
            teacher_serializer = TeacherSerializer(tea_obj).data
            print(teacher_serializer)
            return Response({
                'status':200,
                'message':'查询单个教师成功',
                'result':teacher_serializer
            })
        else:
            teacher_objects_all = Teacher.objects.all()
            tea_data = TeacherSerializer(teacher_objects_all,many=True).data
            print(tea_data)

            return Response({
                'status':200,
                'message':'查询所有教师成功',
                'result':tea_data
            })

    def post(self,request, *args, **kwargs):
        request_data = request.data

        if not isinstance(request_data, dict) or request_data =={}:
            return Response({
                'status':400,
                'message':'参数错误',
            })

        serializer = TeacherDeSerializer(data = request_data)

        if serializer.is_valid():
            tea_ser = serializer.save()
            return Response({
                'status':200,
                'message':'教师添加成功',
                'result':TeacherSerializer(tea_ser).data
            })
        else:
            return Response({
                'status':400,
                'message':'添加教师失败',
                'result':serializer.errors
            })


#80515 hacked temporary version (c) @alex for @oleg FIXME:DEL
import sys
import os

try:
    from link_grammar.parsestat import calc_parse_quality
    from link_grammar.lgparse import traverse_dir
except:
    from parsestat import calc_parse_quality
    from lgparse import traverse_dir

__all__ = ['Load_File', 'Get_Parses', 'MakeSets', 'Evaluate_Parses', 'get_parses', 'eval_parses', 'compare_ull_files']

def Load_File(filename):
    """
        Loads a data file
    """
    with open(filename) as file:
        data = file.readlines()
    return data

def get_parses(data, ignore_wall:bool=True):
    """
        Separates parses from data into format:
        [
          [[sentence-parse1][link1-parse1][link2-parse1] ... ]
          [[sentence-parse2][link1-parse2][link2-parse2] ... ]
          ...
        ]
        Each list is splitted into tokens using space.
    """
    parses = []              # list of parses where each parse consists of two elements: sentence and the set of links
    parse = []

    for line in data:

        # Suppose that sentence line always starts with a letter
        if line[0].isalpha() or line[0] == "[":

            if len(parse) > 0:
                parses.append(parse)
                parse = []

            parse.append((((line.replace("[", "")).replace("]", "")).replace("\n", "")).strip())
            parse.append(set())
            parse.append(int(0))

        # Parses are always start with a digit
        elif line[0].isdigit():
            link = line.split()

            assert len(link) == 4

            # Do not add LW and period links to the set if 'ignore_wall' is specified
            if ignore_wall and (link[1] == "." or link[3] == "."
                                or link[1].startswith(r"###") or link[3].startswith(r"###")):

                parse[2] += 1   # count ignored links
                continue

            # Only token indexes are added to the set
            parse[1].add((link[0], link[2]))

    # Last parse should always be added to the list
    if len(parse) > 0:
        parses.append(parse)

    parses.sort()

    # print(parses, file=sys.stdout)

    return parses

def eval_parses(test_parses:list, ref_parses:list, verbose:bool, ignore=bool, ofile=sys.stdout):
    """
        Compares test_parses against ref_parses link by link
        counting errors
    """
    total_linkages = len(ref_parses)        # in gold standard
    extra_links = 0.0                       # links present in test, but not in ref
    missing_links = 0.0                     # links present in ref, but not in test
    ignored_links = 0.0                     # ignored links, if ignore is active
    quality_ratio = 0.0                     # quality ratio

    if len(ref_parses) != len(test_parses):
        print("Error: files don't contain same parses. "
              "Number of sentences missmatch. Ref={}, Test={}".format(len(ref_parses), len(test_parses)))
        return

    for ref_parse, test_parse in zip(ref_parses, test_parses):

        if ref_parse[0] != test_parse[0]:
            print(ref_parse[0], file=ofile)
            print(test_parse[0], file=ofile)
            print("Error: Something went wrong. Sentences missmatch.", file=ofile)
            return

        # if verbose:
        #     print("Sentence: {}".format(" ".join(ref_sent)), file=ofile)
        #     print("Missing links: {}".format(current_missing), file=ofile)
        #     print("Extra links: {}".format(len(test_sets)), file=ofile)

        (m, e, q) = calc_parse_quality(test_parse[1], ref_parse[1])

        missing_links += m
        extra_links += e
        quality_ratio += q
        ignored_links += test_parse[2]

    if total_linkages > 1:
        missing_links /= float(total_linkages)
        extra_links   /= float(total_linkages)
        quality_ratio /= float(total_linkages)
        ignored_links /= float(total_linkages)

    print("\nParse quality: {0:2.2f}%".format(quality_ratio*100.0), file=ofile)
    print("A total of {} links".format(total_linkages), file=ofile)
    print("Average ignored links: {0:2.2f}".format(ignored_links), file=ofile)
    print("Average missing links: {0:2.2f}".format(missing_links), file=ofile)
    print("Average extra links:  {0:2.2f}".format(extra_links), file=ofile)

def Get_Parses(data, ignore_wall=True):
    """
        Separates parses from data into format:
        [
          [[sentence-parse1][link1-parse1][link2-parse1] ... ]
          [[sentence-parse2][link1-parse2][link2-parse2] ... ]
          ...
        ]
        Each list is splitted into tokens using space.

        Token renumeration added. Works only when ignore_wall=True.
    """
    parses = []
    parse_num = -1
    new_flag = True

    links_skipped = 0

    for line in data:

        if line == "\n":
            new_flag = True
            continue

        if new_flag:
            parse_num += 1
            new_flag = False
            parses.append([[(((line.replace("[", "")).replace("]", "")).replace("\n", "")).strip()]])
            links_skipped = 0
            continue

        parse = line.split()

        assert len(parse) == 4

        if ignore_wall and (parse[1] == "." or parse[3] == "."
                            or parse[1].startswith(r"###") or parse[3].startswith(r"###")):
            links_skipped += 1
            continue

        new_parse = []

        for i, element in zip(range(len(parse)), parse):
            new_element = element if (i & 1) else int(element) # - links_skipped
            new_parse.append(new_element)

        # print(new_parse)
        parses[parse_num].append(new_parse)

    parses.sort()

    # print(parses)

    return parses

def MakeSets(parse):
    """
        Gets a list with links (without full sentence) and
        and makes sets for each link's ids
    """
    link_sets = [{(link[0], link[1]), (link[2], link[3])} for link in parse]
    return link_sets

def Evaluate_Parses(test_parses, ref_parses, verbose, ignore, ofile):
    """
        Compares test_parses against ref_parses link by link
        counting errors
    """
    total_links = 0     # in gold standard
    #extra_links = 0     # links present in test, but not in ref
    missing_links = 0   # links present in ref, but not in test
    ignored_links = 0   # ignored links, if ignore is active

    for ref_parse, test_parse in zip(ref_parses, test_parses):
        current_missing = 0
        ref_sent = ref_parse.pop(0)[0].split()
        test_sent = test_parse.pop(0)[0].split()

        if ref_sent != test_sent:
            print(ref_sent, file=ofile)
            print(test_sent, file=ofile)
            print("Error: files don't contain same parses", file=ofile)

            print(ref_sent)
            print(test_sent)
            print("Error: files don't contain same parses")
            return

        ref_sets = MakeSets(ref_parse)  # using sets to ignore link directions
        test_sets = MakeSets(test_parse)
        sent_length = str(len(ref_sent))

        # loop over every ref link and try to find it in test
        for ref_link in ref_sets:
            if ignore:
                if (('0', '###LEFT-WALL###') in ref_link) or ((sent_length, ".") in ref_link):
                    ignored_links += 1
                    continue
            total_links += 1
            if ref_link in test_sets:
                test_sets.remove(ref_link)
            else:
                current_missing += 1 # count links not contained in test

        if verbose:
            print("Sentence: {}".format(" ".join(ref_sent)), file=ofile)
            print("Missing links: {}".format(current_missing), file=ofile)
            print("Extra links: {}".format(len(test_sets)), file=ofile)

        #extra_links += len(test_sets) # count links not contained in reference
        missing_links += current_missing

    score = 1 - missing_links / total_links
    print("\nParses score: {0:2.2f}".format(score*100.0), file=ofile)
    print("A total of {} links".format(total_links), file=ofile)
    print("A total of {} ignored links".format(ignored_links), file=ofile)
    print("A total of {} missing links".format(missing_links), file=ofile)
    #print("A total of {} extra links".format(extra_links))

def compare_ull_files(test_path, ref_file, verbose, ignore_WALL):
    """ Initiates evaluation process for one or multiple files."""

    def evaluate(test_file):
        """ Callback evaluation function """
        print("\nComparing parses:")
        print("-----------------")
        print("File being tested: '" + test_file + "'")
        print("Reference file   : '" + ref_file + "'")

        out_file = test_file + ".qc"
        print("Result file      : '" + out_file + "'")

        try:
            test_data = Load_File(test_file)
            # test_parses = Get_Parses(test_data)

            test_parses = get_parses(test_data, ignore_WALL)

            ref_data = Load_File(ref_file)
            # ref_parses = Get_Parses(ref_data)

            ref_parses = get_parses(ref_data, ignore_WALL)

            with open(out_file, "w") as ofile:
                print("Reference file   : '" + ref_file + "'", file=ofile)
                # Evaluate_Parses(test_parses, ref_parses, verbose, ignore_WALL, ofile)

                eval_parses(test_parses, ref_parses, verbose, ignore_WALL, ofile)

        except IOError as err:
            print("IOError: " + str(err))

        except Exception as err:
            print("Exception: " + str(err))

    # If specified name is a file.
    if os.path.isfile(test_path):
        evaluate(test_path)

    # If specified name is a directory.
    elif os.path.isdir(test_path):
        traverse_dir(test_path, ".ull2", evaluate, None, True)

    # If file or directory does not exist.
    else:
        raise("Error: File or directory '" + test_path + "' does not exist.")


# ‐*‐ coding: utf‐8 ‐*‐
"""
一个关键词serp上同一个域名出现N个url排名 计算1次，相当于计算首页词数
"""
import requests
from pyquery import PyQuery as pq
import threading
import queue
import time
from urllib.parse import urlparse

class bdpcCover(threading.Thread):

    def __init__(self):
        threading.Thread.__init__(self)

    # 读取txt文件 关键词进入队列
    @staticmethod
    def read_file(filepath):
        q = queue.Queue()
        for kwd in open(filepath,encoding='utf-8'):
            kwd = kwd.strip()
            q.put(kwd)
        return q

    # 获取某词serp源码
    def get_html(self,url,retry=2):
        try:
            r = requests.get(url=url,headers=user_agent,timeout=5)
        except Exception as e:
            print('获取源码失败',e)
            if retry > 0:
                self.get_html(url,retry-1)
        else:
            html = r.text
            return html

    # 获取某词serp源码上自然排名的所有url
    def get_encrpt_urls(self,html):
        encrypt_url_list = []
        if html and '_百度搜索' in html:
            doc = pq(html)
            try:
                a_list = doc('.t a').items()
            except Exception as e:
                print('未提取到serp上的解密url', e, url)
            else:
                for a in a_list:
                    encrypt_url = a.attr('href')
                    if encrypt_url.find('http://www.baidu.com/link?url=') == 0:
                        encrypt_url_list.append(encrypt_url)
        return encrypt_url_list

    # 解密某条加密url
    def decrypt_url(self,encrypt_url,retry=1):
        try:
            encrypt_url = encrypt_url.replace('http://','https://')
            r = requests.head(encrypt_url,headers=user_agent)
        except Exception as e:
            print(encrypt_url,'解密失败',e)
            if retry > 0:
                self.decrypt_url(encrypt_url,retry-1)
        else:
            return r.headers['Location']

    # 获取某词serp源码首页排名真实url
    def get_real_urls(self,encrypt_url_list):
        real_url_list = [self.decrypt_url(encrypt_url) for encrypt_url in encrypt_url_list]
        return real_url_list

    # 提取某条url域名部分
    def get_domain(self,real_url):
        try:
           res = urlparse(real_url)
        except Exception as e:
           print (e,real_url)
           domain = "xxx"
        else:
           domain = res.netloc
        return domain

    # 获取某词serp源码首页排名真实url的域名部分
    def get_domains(self,real_url_list):
            domain_list = [self.get_domain(real_url) for real_url in real_url_list]
            # 搜一个词 同一个域名多个url出现排名 只计算一次
            domain_set = set(domain_list)
            return domain_set

    # 线程函数
    def run(self):
        global success_num
        while 1:
            kwd = q.get()
            url = "https://www.baidu.com/s?ie=utf-8&wd={0}".format(kwd)
            html = self.get_html(url)
            encrypt_url_list = self.get_encrpt_urls(html)
            real_url_list = self.get_real_urls(encrypt_url_list)
            domain_set = self.get_domains(real_url_list)
            if domain_set:
                try:
                    threadLock.acquire()
                    for domain in domain_set:
                        result[domain] = result[domain]+1 if domain in result else 1
                    success_num += 1
                    print('查询成功{0}个'.format(success_num))
                except Exception as e:
                    print(e)
                finally:
                    print (kwd,'查询结束')
                    threadLock.release()
            q.task_done()

    # 保存数据
    @staticmethod
    def save():
        print ('开始save.....')
        res_sort = sorted(result.items(), key=lambda s: s[1], reverse=True)
        print(res_sort)
        with open('result2.txt','w',encoding="utf-8") as f:
            for domain,value in res_sort:
                # print(domain,type(domain),type(str(value)))
                f.write(str(domain)+'\t'+str(value)+'\n')

if __name__ == "__main__":
    start = time.time()

    user_agent = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0'}
    threadLock = threading.Lock()  # 锁
    result = {}   # 初始结果保存字典
    success_num = 0  # 查询成功个数
    q = bdpcCover.read_file('kwd.txt')
    all_num = q.qsize() #总词数

    # 设置线程数
    for i in list(range(5)):
        t = bdpcCover()
        t.setDaemon(True)
        t.start()
    q.join()

    # 结果保存文件
    bdpcCover.save()
    end = time.time()
    print('\n关键词共{0}个,查询成功{1}个,耗时{2}min'.format(all_num,success_num,(end-start)/60) )