from mysql import connector

class Database(object):
    def __init__(self, db_config, debug = False):
        self.config = db_config

        self.debug = debug

    def connect(self):
        try:
            con = connector.connect(**self.config)

            if "AttributeError" in str(type(con)):
                raise Exception("invalid database params in config file")

            cur = con.cursor(dictionary=True, buffered=True)

            return con, cur
        except connector.Error as err:
            if err.errno == connector.errorcode.ER_ACCESS_DENIED_ERROR:
                raise Exception('ACCESS_DENIED_ERROR: credentials missmatch')
            elif err.errno == connector.errorcode.ER_BAD_DB_ERROR:
                raise Exception('BAD_DB: unknown database name')
            else:
               raise Exception(err)

    def get_rows(self, sqlQuery, params=None):
        result = None
        con, cur = self.connect()

        try:
            cur.execute(sqlQuery, params)
            result = cur.fetchall()
            if (result is None):
                raise ValueError(f"No results returned for the query {sqlQuery}")
        except Exception as error:
            if self.debug:
                print(error)
            result = None
        finally:
            con.close()
            cur.close()

            return result

    def get_one_row(self, sqlQuery, params=None):
        con, cur = self.connect()

        try:
            cur.execute(sqlQuery, params)
            result = cur.fetchone()

            if (result is None):
                raise ValueError(f"No results returned for the query {sqlQuery}")
        except Exception as error:
            if self.debug:
                print(error)
            result = None
        finally:
            con.close()
            cur.close()

            return result

    def execute_sql(self, sqlQuery, params=None):
        result = None
        con, cur = self.connect()

        try:
            cur.execute(sqlQuery, params)
            con.commit()

            result = self.cursor.lastrowid #bevestigig van create (int of 0)
            if result != 0:  # is een insert, deze stuur het lastrowid terug.
                result = result
            else:  # is een update of een delete
                if cur.rowcount == -1:  # Er is een fout in de SQL
                    raise Exception(f'Incorrect SQL syntax, {sqlQuery}')
                elif cur.rowcount == 0:  # Er is niks gewijzigd, where voldoet niet of geen wijziging in de data
                    result = 0
                else:  # Hoeveel rijen werden gewijzigd
                    result = cur.rowcount
        except connector.Error as error:
            con.rollback()
            result = None
            print (f"Error: Data was not saved. {error.msg}")
        finally:
            con.close()
            cur.close()

            return result


import argparse

from .isotope import Isotope
from .calc import NMRcalc

desc = '''NMRFREQ - analyzing magnet frequency utility for the NSL'''

class Driver:

    def __init__(self, masstable, version=''):
        self.masstable = masstable
        self.version = version
        self.parser = argparse.ArgumentParser(description=desc)
        self.defineUsage()

    def defineUsage(self):
        p = self.parser
        p.add_argument('-v', '--version', action='version',
                       version='nmrfreq {0}'.format(self.version))
        p.add_argument('-i', '--isotope', type=str, nargs=1,
                       help='desired isotope name')
        p.add_argument('-q', '--charge', type=int, nargs='*',
                       help='selected charge state')
        g = p.add_mutually_exclusive_group()
        g.add_argument('-e', '--energy', type=float, nargs='*',
                       help='desired beam energy [MeV]')
        g.add_argument('-f', '--freq', type=float, nargs=1,
                       help='desired NMR frequency [MHz]')

    def drive(self):
        self.parser.parse_args(namespace=Driver)
        try:
            self.createIsotope()
            self.performCalculation()
        except KeyError:
            print('No matching isotope for {0} found'.format(self.isotope))
        except IndexError:
            print('Isotope {0} not a valid input'.format(self.isotope))
        except TypeError:
            self.parser.print_help()

    def createIsotope(self):
        if (self.isotope is None and
                (self.energy is not None or
                 self.freq is not None or self.field is not None)):
            self.isotope = ['H1']
        self.isotope = self.isotope[0]
        self.isotope = Isotope(self.isotope, self.masstable)

    def performCalculation(self):
        self.calc = NMRcalc(
            self.isotope, self.charge, self.energy, self.freq)
        self.calc.processValues()


# -*-coding:utf-8-*-
import stomp
import time
import base64
import random
import string
import os

queue_name = '/queue/test'
listener_name = 'SampleListener'

class SampleListener(object):
    def on_error(self, headers, message):
        print('错误：', message)
    def on_message(self, headers, message):
        print('接收：', message)
        #  随机字符串
        ran_str = ''.join(random.sample(string.ascii_letters + string.digits, 8))
        img = base64.b64decode(message)
        file = open(os.getcwd() + '/'+ ran_str +'.jpg', 'wb')
        file.write(img)
        file.close()

# 推送到队列queue
def send_to_queue(msg):
    conn = stomp.Connection10([('127.0.0.1', 61613)])
    conn.start()
    conn.connect()
    print('发送：', msg)
    conn.send(queue_name, msg)
    conn.disconnect()

##从队列接收消息
def receive_from_queue():
    conn = stomp.Connection10([('127.0.0.1', 61613)])
    conn.set_listener(listener_name, SampleListener())
    conn.start()
    conn.connect()
    conn.subscribe(queue_name)
    time.sleep(1)  # secs
    conn.disconnect()

if __name__ == '__main__':
     #send_to_queue('len 123')
     # 无限循环，接收队列中的消息
     var = 1
     while var == 1:
         receive_from_queue()


import requests
from bs4 import BeautifulSoup
from flask import current_app
import pandas as pd
import my_util.db_module as dm

def change_date(x):
    y = x.split(' ')
    month = y[1][:-1] if len(y[1][:-1]) == 2 else '0'+y[1][:-1]
    day = y[2][:-1] if len(y[2][:-1]) == 2 else '0'+y[2][:-1]
    return f'{y[0][:-1]}-{month}-{day}'

def get_region_by_date(date):
    with open('D:/workspace/python_flask/04.MachineLearning/static/data/corona_key.txt', mode='r') as key_fd:
        govapi_key = key_fd.read(200)
    start_date = date.replace('-','')
    end_date = date.replace('-','')
    page = 1
    corona_url = 'http://openapi.data.go.kr/openapi/service/rest/Covid19/getCovid19SidoInfStateJson'
    url = f'{corona_url}?ServiceKey={govapi_key}&pageNo={page}&numOfRows=10&startCreateDt={start_date}&endCreateDt={end_date}'

    result = requests.get(url)
    soup = BeautifulSoup(result.text, 'xml')
    resultCode = soup.find('resultCode').get_text()
    if resultCode == '99':
        current_app.logger.info(soup.find('resultMsg').string)
        return
    if resultCode == '00' and soup.find('totalCount').string == '0':
        current_app.logger.info('There is no data!!!')
        return

    items = soup.find_all('item')
    item_count = len(items)
    for index, item in enumerate(items):
        if item_count > 30 and index >= int(item_count/2):
            break
        stdDay = change_date(item.find('stdDay').string)
        deathCnt = int(item.find('deathCnt').string) if item.find('deathCnt') else 0
        defCnt = int(item.find('defCnt').string) if item.find('defCnt') else 0
        gubun = item.find('gubun').string
        incDec = int(item.find('incDec').string)
        isolClearCnt = int(item.find('isolClearCnt').string) if item.find('isolClearCnt') else 0
        isolIngCnt = int(item.find('isolIngCnt').string) if item.find('isolIngCnt') else 0
        localOccCnt = int(item.find('localOccCnt').string) if item.find('localOccCnt') else 0
        overFlowCnt = int(item.find('overFlowCnt').string) if item.find('overFlowCnt') else 0
        qurRate = None
        if item.find('qurRate'):
            qur = item.find('qurRate').string
            if qur != None and qur.count('.') == 2:
                qur = qur[:-1]
            if qur != None and qur[0] in '0123456789':
                qurRate = float(qur)

        params = [stdDay, deathCnt, defCnt, gubun, incDec, isolClearCnt, isolIngCnt,
                localOccCnt, overFlowCnt, qurRate]
        dm.write_region(params)

    current_app.logger.info(f'{date} region data successfully inserted.')

def get_agender_by_date(date):
    with open('D:/workspace/python_flask/04.MachineLearning/static/data/corona_key.txt', mode='r') as key_fd:
        govapi_key = key_fd.read(200)
    start_date = date.replace('-','')
    end_date = date.replace('-','')
    page = 1
    corona_url = 'http://openapi.data.go.kr/openapi/service/rest/Covid19/getCovid19GenAgeCaseInfJson'
    url = f'{corona_url}?ServiceKey={govapi_key}&pageNo={page}&numOfRows=10&startCreateDt={start_date}&endCreateDt={end_date}'

    result = requests.get(url)
    soup = BeautifulSoup(result.text, 'xml')
    resultCode = soup.find('resultCode').get_text()
    if resultCode == '99':
        current_app.logger.info(soup.find('resultMsg').string)
        return
    if resultCode == '00' and soup.find('totalCount').string == '0':
        current_app.logger.info('There is no data!!!')
        return

    items = soup.find_all('item')
    for item in items:
        createDt = item.find('createDt').string.split(' ')[0]
        confCase = int(item.find('confCase').string)
        confCaseRate = float(item.find('confCaseRate').string)
        death = int(item.find('death').string)
        deathRate = float(item.find('deathRate').string)
        criticalRate = float(item.find('criticalRate').string)
        gubun = item.find('gubun').string
        seq = int(item.find('seq').string)
        updateDt = item.find('updateDt').string

        params = [createDt, confCase, confCaseRate, death, deathRate, criticalRate,
                    gubun,seq,updateDt]
        dm.write_agender(params)

    current_app.logger.info(f'{date} agender data successfully inserted.')


import logging

from django.db import transaction

from .models import Task, delay
from .task import SerialTask, TaskFunc
from .backends import backend

logger = logging.getLogger('cq')

def run_task(message):
    try:
        task_id = message['task_id']
    except (TypeError, KeyError):
        logger.error('Invalid CQ message.')
        return
    task = Task.objects.get(id=task_id)
    func_name = task.signature['func_name']
    if task.status == Task.STATUS_REVOKED:
        logger.info('Not running revoked task: {}'.format(func_name))
        return
    logger.info('Running task: {}'.format(func_name))
    backend.set_current_task(task_id)
    task.pre_start()
    taskFunc = TaskFunc.get_task(func_name)
    if taskFunc.atomic:
        with transaction.atomic():
            _do_run_task(task)
    else:
        _do_run_task(task)

def _do_run_task(task):
    try:
        result = task.start(pre_start=False)
    except Exception as err:
        task.failure(err)
    else:
        if isinstance(result, Task):
            task.waiting(task=result)
        else:
            if isinstance(result, SerialTask):
                result = result.result
            if task.subtasks.exists():
                task.waiting(result=result)
            else:
                task.success(result)
    finally:
        backend.set_current_task()


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# Author:X.Ning <ngx@ngx.wiki>

li = ['alex','eric','rain']

# a. 计算长度并且输出
li_a = len(li)
print(li_a)

# b. 列表中追加元素“seven”，并且输出添加后的列表
li.append('seven')
print(li)

# c.


# 2018.01.26 Learning_rate too large cause the gradient explosion
import numpy as np
import matplotlib.pyplot as plt

np.random.seed(1)
class Linear_regression(object):
    def __init__(self):
        self.w=None
        self.b=None

    def fit(self,x,y):
        batch_size,feature_size=x.shape
        self.w=np.random.random((feature_size, 1))
        self.b=np.random.random(1)
        self._gradient_descent(x,y)

    def eval(self):
        pass

    def predict(self,x):
        y_hat=np.dot(x,self.w)+self.b
        return y_hat

    def plot(self,x,y,y_pred):
        plt.plot(x, y, 'r-')
        plt.plot(x, y_pred, 'b.')
        plt.show()

    def _gradient_descent(self,x,y,learning_rate=0.0001,epoch=100):
        for i in range(epoch):
            y_hat=self.predict(x)  # => batch_size*1
            loss=np.dot((y_hat-y).T,(y_hat-y))
            w_gradient = np.dot(x.T, x).dot(self.w) - np.dot(x.T, (y-self.b))
            b_gradient=self.b-np.mean(y-np.dot(x,self.w))
            self.w = self.w - learning_rate * w_gradient
            self.b=self.b-learning_rate*b_gradient
            print('step:', i, 'Loss:', loss[0][0])

    def __str__(self):
        return 'weights\t:%s\n bias\t:%f\n' % (self.w, self.b)

def generate_data():
    '''generate the examples to implement linear regression by numpy
    #arguments: None
    #return: x and y
    x.shape=[],y.shape=[]'''
    x = np.linspace(1, 30, num=30).reshape(-1, 3)
    x1 = np.ones(10).reshape(-1, 1)
    x_extended = np.column_stack((x, x1))
    y = np.linspace(1, 10, num=10) + np.random.random(10)
    y = y.reshape(-1, 1)
    return x,y

if __name__ == "__main__":
    x, y = generate_data()
    lr = Linear_regression()
    lr.fit(x, y)
    y_hat = lr.predict(x)
    lr.plot(x, y, y_hat)
    print(y)
    print(y_hat)
    print(lr)


import numpy as np
from NN import NeuralNetwork

# Input dataset
X = np.array([
    [0, 0, 1],
    [0, 1, 1],
    [1, 0, 1],
    [1, 1, 1],
    [1, 1, 0],
    [0, 0, 0]
])

# Output dataset
y = np.array([[0,0,1,1,0,1]]).T

nn = NeuralNetwork(3, 4, 1)

nn.train(X, y)

print(nn.guess(X))


import string
#login user
# register user
#logout user
# account update UserForm)
import random
import os
from flask import render_template,url_for,flash,redirect,request,Blueprint,abort
from flask_login import login_user,current_user,logout_user,login_required
from werkzeug.security import generate_password_hash
from werkzeug.utils import secure_filename
from cms import db,basedir
from cms.models import Attachment, Request, User,Course
from cms.users.forms import RegistrationForm,LoginForm,UpdateUserForm, requestCourseForm
users=Blueprint('users',__name__)

@users.route('/register',methods=['GET', 'POST'])
def register():
    form=RegistrationForm()
    if form.validate_on_submit():
        # user_data={k:v.data for k,v in form}
        # print(user_data)
        print("name is",form.name.data)
        user=User(form.name.data,form.email.data,form.password.data,form.year.data,form.branch.data)
        db.session.add(user)
        db.session.commit()
        flash('Thanks for registration!')
        return redirect(url_for('users.login'))
    return render_template('register.html',form=form)

@users.route('/login',methods=['GET', 'POST'])
def login():
    form=LoginForm()
    if form.validate_on_submit():
        user =User.query.filter_by(email=form.email.data).first()
        if user is not None and user.check_password(form.password.data):
            login_user(user)
            flash('Login success')
            next=request.args.get('next')
            if next==None or not next[0]=='/':
                next=url_for('core.index')
            return redirect(next)

    return render_template('login.html',form=form)

@users.route('/logout')
def logout():
    logout_user()
    return redirect(url_for("core.index"))

@users.route('/account',methods=['GET', 'POST'])
@login_required
def account():
    form=UpdateUserForm()
    if form.validate_on_submit():

        if form.email.data:
            current_user.email=form.email.data
        if form.password.data:
            current_user.password_hash = generate_password_hash(form.password.data)
        db.session.commit()
        flash('User account updated')
        return redirect(url_for('users.account'))
    elif request.method=="GET":
        form.email.data=current_user.email
    return render_template('account.html',form=form)

@users.route('/enroll',methods=['GET', 'POST'])
@login_required
def enroll():
    avail_courses=[a for a in Course.query.filter_by(can_apply=True).all() if a not in current_user.courses and current_user.branch.name in [b.name for b in a.branches] ]
    not_eligible_courses=[a for a in Course.query.filter_by(can_apply=True).all() if a not in current_user.courses and current_user.branch.name not in [b.name for b in a.branches] and a not in current_user.requested_courses ]
    requests = current_user.requests
    if requests:
        print(requests)
    form = requestCourseForm()
    if form.submit.data and form.validate:
        attachments = []
        if not current_user.is_authenticated:
            abort(405)
        newCourseNote = Request(current_user.id,form.course_id.data,form.title.data,form.details.data)
        db.session.add(newCourseNote)
        db.session.commit()

        if form.attachments.data:
            for uploaded_file in request.files.getlist('attachments'):

                filename, file_extension = os.path.splitext(
                    uploaded_file.filename)
                if not filename or not file_extension:
                    continue

                savename = secure_filename(filename)+''.join(
                    random.choice(string.ascii_lowercase) for i in range(16))+file_extension
                print(filename, savename)
                if savename == "":
                    break

                uploaded_file.save(os.path.join(
                    basedir, '..', '..', 'static_material', savename))
                new_attachment = Attachment(
                    filename, file_extension, url_for('course.serve_file', filename=savename), request_id=newCourseNote.id)
                attachments.append(new_attachment)
        db.session.add_all(attachments)
        db.session.commit()
        return redirect(url_for('users.enroll'))

    return render_template('enroll.html', avail_courses=avail_courses, not_eligible_courses=not_eligible_courses, requestForm=form, requests=current_user.requests)

@users.route('/enroll/<course_id>')
@login_required
def enroll_course(course_id):
    courseToAdd=Course.query.filter_by(id=course_id).first()
    if courseToAdd is None or courseToAdd in current_user.courses:
        flash("Course cannot be added")
        abort(405)
    current_user.courses.append(courseToAdd)
    db.session.commit()
    return redirect(url_for('core.index'))

@users.route('/request/<course_id>')
@login_required
def request_course(course_id):
    courseToAdd = Course.query.filter_by(id=course_id).first()
    if courseToAdd is None or courseToAdd in current_user.courses:
        flash("Course cannot be added")
        abort(405)
    newRequest=Request(current_user.id,course_id)
    db.session.add(newRequest)
    db.session.commit()
    return redirect(url_for('users.enroll'))


#!/usr/bin/env python
# coding=UTF-8
'''
Author: Wei Luo
Date: 2021-07-31 20:42:56
LastEditors: Wei Luo
LastEditTime: 2021-08-15 23:20:25
Note: Note
'''

import rospy
import numpy as np
from itm_mav_msgs.msg import itm_trajectory_msg, itm_trajectory_point
from itm_mav_srvs.srv import itm_trajectory_srv
from geometry_msgs.msg import PoseStamped
from nav_msgs.msg import Odometry

# global parameters
robot_pose = np.array([])
got_robot_pose = False
got_robot_odom = False

def robot_odom_callback(msg):
    global robot_pose, got_robot_odom
    if not got_robot_odom:
        got_robot_odom = True
    robot_pose = np.array(
        [msg.pose.pose.position.x, msg.pose.pose.position.y, msg.pose.pose.position.z])

def robot_pose_callback(msg):
    global robot_pose, got_robot_pose
    if not got_robot_pose:
        got_robot_pose = True
    robot_pose = np.array(
        [msg.pose.position.x, msg.pose.position.y, msg.pose.position.z])

if __name__ == '__main__':
    rospy.init_node('uav_squared_trajectory')
    rospy.wait_for_service('/itm_trajectory_srv')
    set_traj_srv = rospy.ServiceProxy('itm_trajectory_srv', itm_trajectory_srv)
    robot_odom_sub_ = rospy.Subscriber(
        '/robot_pose', Odometry, robot_odom_callback)
    robot_pose_sub_ = rospy.Subscriber(
        '/vrpn_client_node/ITM_Q330/pose', PoseStamped, robot_pose_callback)

    trajectory_waypoints = np.array([
        [0.0, 0.0, 0.5],
        [0.5, 0.0, 0.5],
        [0.5, 0.5, 0.5],
        [0.0, 0.5, 0.5],
        [-0.5, 0.5, 0.5],
        [-0.5, 0.0, 0.5],
        [-0.5, -0.5, 0.5],
        [0.0, -0.5, 0.5],
        [0.5, -0.5, 0.5],
        [0.5, 0.0, 0.5],
        [0.0, 0.0, 0.5],
    ])

    timestamp_list = [
        1.0, 3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0, 17.0, 19.0, 20.0, 24.0,
    ]

    # main loop
    trajectory_list = []
    msg_index = 0
    publish_index = 0

    for idx in range(trajectory_waypoints.shape[0]):

        point = itm_trajectory_point()
        point.x = trajectory_waypoints[idx, 0]
        point.y = trajectory_waypoints[idx, 1]
        point.z = trajectory_waypoints[idx, 2]
        point.yaw = 0.0
        point.derivative = 0
        point.fixed = True
        point.time_known = True
        point.time_stamp = timestamp_list[idx]
        trajectory_list.append(point)

    while not rospy.is_shutdown():
        if (not got_robot_pose and not got_robot_odom):
            rospy.logerr("no robot pose or odometry information received")
        elif (got_robot_odom and got_robot_pose):
            rospy.logerr(
                "You've received both odom (sim) and pose (OptiTrack). That is impossible and unsupported! Stop and check your setup")
            break
        else:
            srv_obj = set_traj_srv(msg_index, trajectory_list)
            if srv_obj.success:
                break


IN_FILEPATH = "in\\01-"
OUT_FILEPATH = "out\\01-"
FILETYPE = "txt"
FILE_NUM = 20

for num in range(1, FILE_NUM + 1):

	path = IN_FILEPATH + '{:0>2}'.format(num) + "." + FILETYPE

	with open(path) as f:
		data = f.readlines()

	data = [ line.replace("\n", "") for line in data ][1]
	result = data.replace("joi", "JOI")

	path = OUT_FILEPATH + '{:0>2}'.format(num) + "." + FILETYPE

	with open(path, mode="w") as f:
		f.write(result + "\n")


import pandas as pd
import ast
from fuzzywuzzy import fuzz

def merge_df(df_drugs_dot_com, merge_with_df,df_name):
    if df_name == 'WebMD':
        jaccard_df = pd.read_csv('./../../dataset/webmd/webmd_v03_drugs_com_jaccard.csv')
    if df_name == 'DrugLib':
        jaccard_df = pd.read_csv('./../../dataset/druglib/druglib_v03_drugs_com_jaccard.csv')

    count_unique_conditions(jaccard_df, merge_with_df,df_name)

    df_merge1 = pd.merge(jaccard_df, df_drugs_dot_com, left_on='id1', right_on='id')
    df_merge2 = pd.merge(df_merge1, merge_with_df, left_on='id2', right_on='id')
    df_merge = df_merge2[['id1', 'joinKey1', 'id2', 'joinKey2', 'drug', 'condition', 'Drug', 'Condition']].rename(
        columns={'drug': 'DrugsCom_drugname', 'condition': 'DrugsCom_condition', 'Condition': df_name + '_condition', 'Drug':df_name + '_drugname'})

    df_merge[df_name + '_New_Condition'] = df_merge.apply(lambda row: ' ', axis=1)
    df_merge['Fuzz_ratio'] = df_merge.apply(lambda row: ' ', axis=1)

    for i in range(len(df_merge)):
        if ast.literal_eval(str(df_merge['joinKey1'][i]))[0] == ast.literal_eval(str(df_merge['joinKey2'][i]))[0]:
            df_merge[df_name + '_New_Condition'][i] = df_merge['DrugsCom_condition'][i]
            df_merge['Fuzz_ratio'][i] = fuzz.ratio(df_merge[df_name + '_condition'][i].lower(),
                                                   df_merge[df_name + '_New_Condition'][i].lower())
        else:
            df_merge[df_name + '_New_Condition'][i] = df_merge[df_name + '_condition'][i]
            df_merge['Fuzz_ratio'][i] = fuzz.ratio(df_merge[df_name + '_condition'][i].lower(), df_merge[df_name + '_New_Condition'][i].lower())

    return df_merge

def count_unique_conditions(jaccard_df,merge_with_df,df_name):
    unique_drugs = set()

    for i in range(len(jaccard_df)):
        if ast.literal_eval(str(jaccard_df['joinKey1'][i]))[0] == ast.literal_eval(str(jaccard_df['joinKey2'][i]))[0]:
            unique_drugs.add(ast.literal_eval(str(jaccard_df['joinKey1'][i]))[0])

    print('No. of unique matching drugs in jaccard of drugs.com and ' + df_name + ": ", len(unique_drugs))
    print(len(merge_with_df))
    count = 0
    condition_list = set()
    for j in range(len(merge_with_df)):
        drug = str(merge_with_df['Drug'][j])
        if drug.split(" ")[0] in list(unique_drugs):
            condition_list.add(merge_with_df['Condition'][j])
            count = count + 1

    print('No. of unique matching conditions in jaccard of drugs.com and ' + df_name + ": ",len(condition_list))

def fuzzy_matching():
    """
        Merge webmd with drugs_dot_com
        Merge druglib with drugs_dot_com
    """

    df_drugs_dot_com = pd.read_csv('./../../dataset/drugs_com/drug_com_v02_conditions_grouped.csv')
    df_webMD = pd.read_csv('./../../dataset/webmd/webmd_v02_conditions_grouped.csv')
    df_drugLib = pd.read_csv('./../../dataset/druglib/druglib_v02_conditions_grouped.csv')

    df_webmd_drugsCom = merge_df(df_drugs_dot_com, df_webMD, "WebMD")
    df_webmd_drugsCom.to_csv('./../../dataset/webmd/webmd_v04_fuzzy_on_conditions.csv', index=False)

    df_druglib_drugs_com = merge_df(df_drugs_dot_com, df_drugLib, "DrugLib")
    df_druglib_drugs_com.to_csv('./../../dataset/druglib/druglib_v04_fuzzy_on_conditions.csv', index=False)


from easydict import EasyDict as edict
import math

cfg = edict()
cfg.DATA_PATH = '/home/sasi/datavault/tools/vault1/model_data/semanticKitti/dataset/sequences'

# Training params
cfg.DEBUG = True
cfg.BATCH_SIZE = 8

cfg.MAX_GRAD_NORM = 1.0
cfg.MOVING_AVERAGE_DECAY = 0.999
cfg.BATCHNORM_DECAY = 0.999
cfg.WEIGHT_DECAY = 0.001
cfg.MOVING_AVERAGE_FOR_VAL = True

# Data trimming params
cfg.MIN_X = -1.
cfg.MAX_X = 1.
cfg.MIN_Y = -1.
cfg.MAX_Y = 1.

# Data range to be considered to normalize b/w [-1, 1]
# eg: default [-75, 75] is normalized to [-1, 1]
cfg.MIN_RANGE_X = -75.
cfg.MAX_RANGE_X = 75.
cfg.MIN_RANGE_Y = -75.
cfg.MAX_RANGE_Y = 75.
cfg.MIN_RANGE_Z = -5.
cfg.MAX_RANGE_Z = 5.

# Grid size of each voxel
# cfg.GRID_SIZE = 1.
cfg.GRID_SIZE = 0.01 # Input is normalized to lie in [-1, 1]

# Optimization params
cfg.ITER_NUM = 300
cfg.SUB_ITER_NUM = 1000
cfg.MOMENTUM = 0.9
cfg.LEARNING_RATE = 0.001

# lr schedule
# changed to cosine with warmup

# Data Augmentation
cfg.RAND_INTENSITY_VARIATION = 0.05 # if intensity lies in [0,1]
cfg.SCALE_PROB = 0.5
cfg.MAX_SCALE_CHANGE = 0.2
cfg.MAX_ROTATION = 180
cfg.MAX_SHIFT_X = 25
cfg.MAX_SHIFT_Y = 25
cfg.MAX_Z_CHANGE = 2

# Loading multiple frames
cfg.SEQ_NUM = 1

def get_x_num(cfg):
    return int(math.floor(
        (cfg.MAX_X - cfg.MIN_X) / cfg.GRID_SIZE)) # + 1 --> why

def get_y_num(cfg):
    return int(math.floor(
        (cfg.MAX_Y - cfg.MIN_Y) / cfg.GRID_SIZE)) # + 2   # add 2 here to seperate different batches -->what's meaning?


# coding:utf-8

from flask import Flask, render_template, request, redirect, url_for, make_response,jsonify
from werkzeug.utils import secure_filename
import time
from PIL import Image
from datetime import timedelta
import os, json, cv2, random, io
import numpy as np
import torch
from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog
from detectron2.data import DatasetCatalog
MetadataCatalog.get("dla_train").thing_classes = ['caption', 'figure', 'page', 'table', 'title', 'text']

# 输入是一个图片的地址，输出为一张图片，可以直接把输出通过imwrite保存。
def inference(input_path,model,model_weight):

    im = cv2.imread(input_path)
    #im = input
    #这里的im需要是一张图片，因此如果是图片路径就需要先通过imread变成图片，如果是url就需要通过load方法。

    cfg = get_cfg()
    cfg.merge_from_file(model)
    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 # set threshold for this mode
    cfg.MODEL.WEIGHTS = model_weight
    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 6

    if torch.cuda.is_available():
        print('we use cuda!')
        cfg.MODEL.DEVICE='cuda'
    else:
        print('running on cpu')
        cfg.MODEL.DEVICE='cpu'

    predictor = DefaultPredictor(cfg)
    outputs = predictor(im)
    outputs["instances"].pred_classes
    outputs["instances"].pred_boxes

    v = Visualizer(im[:,:,::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)
    out = v.draw_instance_predictions(outputs["instances"].to("cpu"))
    outs = out.get_image()[:, :, ::-1]

    if os.path.exists(input_path):
        os.remove(input_path)
        print('remove image')

    if os.path.splitext(input_path)[-1] == ".jpg":
        cv2.imwrite(input_path,outs)
        print('input is a jpg file:',outs)
        return input_path
    else:
        image_name = os.path.splitext(os.path.split(input_path)[1])[0]
        print('image name:',image_name)
        jpg_name = os.path.join(os.path.split(input_path)[0],image_name+'.jpg')
        cv2.imwrite(jpg_name,outs)
        print('convert input to jpg:',jpg_name)
        return jpg_name

#input = "demo/input1.jpg"
model = "configs/DLA_mask_rcnn_X_101_32x8d_FPN_3x.yaml"
model_weight = 'model_weight/pub_model_final.pth'

#设置允许的文件格式
ALLOWED_EXTENSIONS = set(['PNG', 'JPG', 'JPGE', 'PBM','PDF'])

def allowed_file(filename):
    ext = filename.rsplit('.', 1)[1]
    return '.' in filename and ext.upper() in ALLOWED_EXTENSIONS

app = Flask(__name__)

# 设置静态文件缓存过期时间
#app.send_file_max_age_default = timedelta(seconds=1)

def resize_image(files):
    im = Image.open(files)
    (w,h) = im.size
    n_w = 500
    n_h = int(h/w*n_w)
    return n_w,n_h

@app.route('/upload', methods=['POST', 'GET'])  # 添加路由
def upload():
    if request.method == 'POST':
        file = request.files['file']

        # 如果非法的拓展名，或者为空，或者没有.那么返回error。
        if not (file and allowed_file(file.filename)):
            #return jsonify({"error": "please check the input form, only accept image file and PDF."})
            return render_template('upload_start2.html',warning = "Illegal input, please choose again.")
        # 根据当前文件所在路径，创建一个储存image的文件夹
        basepath = os.path.dirname(__file__)
        file_path = os.path.join(basepath, 'static/result')
        print('file path:',file_path)
        if not os.path.exists(file_path):
            os.makedirs(file_path, 755)

        # 保存图片
        file_name = secure_filename(file.filename)
        upload_path = os.path.join(file_path, file_name)
        file.save(upload_path)
        print('file path:',file_path,'file name:',file_name,'upload path:',upload_path)

        # 推断结果，并保存
        infer_path = inference(upload_path,model,model_weight)
        infer_name = os.path.split(infer_path)[1]

        # 重新调整图片大小，使得适合屏幕
        n_w,n_h = resize_image(infer_path)
        print('new size is:',n_w,n_h,'file name is:',file_name)

        return render_template('upload_done2.html', input_name = infer_name, new_weight = n_w, new_height = n_h)

    return render_template('upload_start2.html')

if __name__ == '__main__':
    # app.debug = True
    app.run(host='0.0.0.0', port=5000)


import json
import math
import numpy as np
import random
from collections import defaultdict, namedtuple

NUM_NODES = 10
NUM_CHANNELS = 30

def popup_node_with_channel(
        visible_nodes,
        hidden_nodes,
        node_to_unichannels
):
    # First attempt following channels of visible nodes at random.
    i = 0
    while i < 10:
        i += 1
        rand = random.randint(0, len(visible_nodes) - 1)
        rand_visible_node = visible_nodes[rand]
        for unichannel in node_to_unichannels[rand_visible_node]:
            if unichannel.partner in hidden_nodes:
                return unichannel.partner, unichannel.channel

    # Alternatively, attempt finding a channel from a hidden node to a visible one.
    i = 0
    while i < 10:
        i += 1
        rand = random.randint(0, len(hidden_nodes) - 1)
        rand_hidden_node = hidden_nodes[rand]
        for unichannel in node_to_unichannels[rand_hidden_node]:
            if unichannel.partner in visible_nodes:
                return rand_hidden_node, unichannel.channel

    print('No hidden node with connection to visible node found. Giving up.')

    return None, None

def popup_channel(visible_nodes, hidden_channels, channels):
    i = 0
    while i < 10:
        i += 1
        rand = random.randint(0, len(hidden_channels) - 1)
        rand_hidden_channel = hidden_channels[rand]
        channel = channels[rand_hidden_channel]
        if channel[0] in visible_nodes and channel[1] in visible_nodes:
            return rand_hidden_channel

    print('No hidden channel with connection between visible nodes found. Giving up.')

    return None

def flash_channel(visible_channels):
    return random.sample(visible_channels, 1)[0]

def generate_nodes():
    """
    Generate random coordinates on the surface of a sphere.
    """
    nodes = np.random.randn(3, NUM_NODES)
    nodes /= np.linalg.norm(nodes, axis=0)
    return list(nodes.transpose().tolist())

def generate_channels(num_nodes):
    """
    Generate random index pairs to connect nodes via channels.
    """
    channels = set()
    for i in range(NUM_CHANNELS):
        channels.add(frozenset(random.sample(range(num_nodes), 2)))

    return [tuple(channel) for channel in channels]

def generate_animations(num_nodes, channels):
    # Map nodes to their channels.
    Unichannel = namedtuple('UniChannel', ['channel', 'partner'])
    node_to_unichannels = defaultdict(list)
    for i, channel in enumerate(channels):
        node_to_unichannels[channel[0]].append(Unichannel(channel=i, partner=channel[1]))
        node_to_unichannels[channel[1]].append(Unichannel(channel=i, partner=channel[0]))

    Animation = namedtuple('Animation', [
        'start_frame',
        'animation_type',
        'element_type',
        'element_id',
        'transfer_id'
    ])

    # Show nodes through random connections.
    animations = []
    visible_nodes = []
    hidden_nodes = list(range(num_nodes))
    visible_channels = []
    hidden_channels = list(range(len(channels)))
    popup_delay = 10

    first_node = random.randint(0, num_nodes - 1)
    first_animation = Animation(
        start_frame=0,
        animation_type='show',
        element_type='node',
        element_id=first_node,
        transfer_id=-1
    )
    animations.append(first_animation)
    hidden_nodes.remove(first_node)
    visible_nodes.append(first_node)

    last_frame = 0
    transfer_id = 0
    while hidden_nodes or hidden_channels:
        # Find a node with a channel to a visible node.
        node = None
        channel = None
        type = None

        rand = random.randint(0, 2)
        if hidden_nodes and rand == 0:
            node, channel = popup_node_with_channel(
                visible_nodes,
                hidden_nodes,
                node_to_unichannels
            )
            type = 'show'
        elif rand == 1:
            channel = popup_channel(visible_nodes, hidden_channels, channels)
            type = 'show'
        elif visible_channels:
            channel = flash_channel(visible_channels)
            type = 'flash'

        # Create animation.
        if node is not None or channel is not None:
            popup_delay = int(math.ceil(0.95 * popup_delay))
            last_frame = last_frame + popup_delay
        if node is not None:
            node_animation = Animation(
                start_frame=last_frame,
                animation_type=type,
                element_type='node',
                element_id=node,
                transfer_id=transfer_id if type == 'flash' else -1
            )
            animations.append(node_animation)
            if type == 'show':
                hidden_nodes.remove(node)
                visible_nodes.append(node)
        if channel is not None:
            channel_animation = Animation(
                start_frame=last_frame,
                animation_type=type,
                element_type='channel',
                element_id=channel,
                transfer_id=transfer_id if type == 'flash' else -1
            )
            animations.append(channel_animation)
            if type == 'show':
                hidden_channels.remove(channel)
                visible_channels.append(channel)

        if type == 'flash':
            transfer_id += 1

    return animations

def run():
    nodes = generate_nodes()
    channels = generate_channels(len(nodes))

    # Write network topography to network file.
    with open('network.json', 'w') as network_file:
        json.dump({'nodes': nodes, 'channels': channels}, network_file, indent=2)

    animations = generate_animations(len(nodes), channels)

    with open('animation.json', 'w') as animation_file:
        json.dump(animations, animation_file, indent=2)

if __name__ == "__main__":
    run()


import json
import numpy as np
import re
import string
from nltk.corpus import stopwords
from collections import Counter
from collections import defaultdict
from datetime import datetime, timedelta
from email.utils import parsedate_tz
import os
import bz2
import bz2file
import time
import scipy.sparse as sps
import pandas as pd
import networkx as nx
import matplotlib.pyplot as plt
from scipy.special import comb
from tqdm import tqdm
import collections

results = re.compile(r'https://[a-zA-Z0-9.?/&=:]*|http://[a-zA-Z0-9.?/&=:]*',re.S)

#read stop words
f = open('stopwords.txt','r')
stop = []
for line in f:
    stop.append(line[:-1])
f.close()

regex_str = [
    r'<[^>]+>', # HTML tags
    r'(?:@[\w_]+)', # @-mentions
    r"(?:\#+[\w_]+[\w\'_\-]*[\w_]+)", # hash-tags
    r'http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-f][0-9a-f]))+', # URLs
    r'(?:(?:\d+,?)+(?:\.?\d+)?)', # numbers
    r"(?:[a-z][a-z'\-_]+[a-z])", # words with - and '
    r'(?:[\w_]+)', # other words
    r'(?:\S)' # anything else
]

tokens_re = re.compile(r'('+'|'.join(regex_str)+')', re.VERBOSE | re.IGNORECASE)

def preprocess(s):
    '''
    tokenize the sentence
    '''
    tokens = tokens_re.findall(s)
    return tokens

def to_datetime(datestring):
    '''
    change a string to datatime format
    '''
    time_tuple = parsedate_tz(datestring.strip())
    dt = datetime(*time_tuple[:6])
    return dt - timedelta(seconds=time_tuple[-1])

def read_bz2(inpath, keyword, word_igonre):
    '''
    read a single bz2 file

    input:
    inpath: the location of the path
    keyword: the entity that you want to filter
    word_igonre: ignore the tweets containing one of the words in word_ignore

    output:
    terms_stop: the list of tokenized tweets after removing stop words, the element is a tweet after tokenization
    dates_: the published time of the tweets
    whole_tweets: the corresponding original tweets
    durre: the time for reading the file
    '''
    infile = bz2.BZ2File(inpath, 'r')
    terms_stop = []
    dates_ = []
    whole_tweets = []
    begin = time.time()
    num_tweet = 0
    for line in infile:
        try:
            tweet = json.loads(line)
        except ValueError:
            continue
        num_tweet = num_tweet + 1
        if 'text' in tweet:
            text = tweet["text"].lower()
            if keyword.lower() not in text:
                continue
            if 'http' in text or 'https' in text:
                continue
            train = results.sub(r'',text)
            train = re.sub('[^A-Za-z#@0-9]+', ' ', train)
            x = [term.lower() for term in preprocess(train)
                               if term.lower() not in stop and not term.startswith(('#', '@'))
                              and not term.isdigit()
                              and len(term)>2]
            flag = 0
            for word in word_igonre:
                if word in x:
                    flag =1
                    break
            if flag == 1:
                continue
            if len(x) < 4:
                continue
            terms_stop.append(x)
            whole_tweets.append(tweet["text"])
            dates_.append(to_datetime(tweet['created_at']))
    end_time =  time.time()
    duree = end_time - begin
    infile.close()
    return terms_stop, dates_, whole_tweets, num_tweet, duree

def read_folder(path, keyword,list_igore, num = False):
    '''
    This is similar to read_bz2. Read all files in a folder
    '''
    files= os.listdir(path)
    if num == False or num > len(files):
        num = len(files)
    terms_stop = []
    dates = []
    whole_tweets = []
    duree = 0
    num_tweet = 0

    for fname in tqdm(range(num)):
        fname = path + "/" + files[fname]
        terms_stop_, dates_, whole_tweets_, num_tweet_, duree_ = read_bz2(fname, keyword, list_igore)
        terms_stop.extend(terms_stop_)
        dates.extend(dates_)
        whole_tweets.extend(whole_tweets_)
        duree = duree + duree_
        num_tweet = num_tweet + num_tweet_
    return terms_stop, dates, whole_tweets, num_tweet, duree

def find_burst(num, accumulated_num, terms_stop, threshold, first_n = False):
    '''
    Given the numbers of tweets for every period, find the burst

    input:
    num:the numbers of tweets for every period by funtion split_by_time
    accumulated_num: accumulated numbers until this period by funtion split_by_time
    terms_stop: the list of tokenized tweets from read_bz2 function
    threshold: only detect burst beyond this threshold, it is multiplied by the average value

    output:
    ind_burst: a dictionnary indicating the burst burst and the burst end
    list_count: a list indicating all words in a given burst
    '''
    ind_burst = search_maxima(num, threshold, first_n)
    begin_burst = ind_burst.keys()
    end_burst = ind_burst.values()
    list_count = []
    for i in range(len(begin_burst)):
        count_all = Counter()
        for j in range(accumulated_num[begin_burst[i]], accumulated_num[end_burst[i]+1]):
            count_all.update(terms_stop[j])
        list_count.append(count_all)

    return ind_burst, list_count

def search_maxima(value_list, threshold,  first_n = False):
    '''
    find local maxima which is bigger than mean value * threshold.
    the neighbors of a maximum is also considred to be in the burst if their difference is very small

    input:
    value_list: the numbers for a period
    threshold: the burst below mean value * threshold is ignored
    first_n: Only find the first N maxima

    output:
    bursts: a dictionnary indicating the burst burst and the burst end
    '''

    threshold = np.mean(value_list) * threshold
    len_ = len(value_list)
    local_max = []

    for i in range(1, len_-1):
        if value_list[i] > threshold and value_list[i] > value_list[i-1] and value_list[i] > value_list[i+1]:
            local_max.append(i)

    bursts = {}
    diff = max(value_list) - min(value_list)

    for pic in local_max:
        for j in range(pic + 1, len_):
            if value_list[j] < threshold or abs(value_list[j] - value_list[pic]) > 0.01 * diff :
                break
        tmp = 0
        for k in range(pic-1,0,-1):
            if value_list[k] < threshold or abs(value_list[k] - value_list[pic]) > 0.01 * diff :
                tmp = k + 1
                break
        bursts[tmp] = j-1

    if 0 not in bursts.keys():
        if value_list[0] > threshold and value_list[0] > value_list[1]:
            bursts[0] = 0
    if len_-1 not in bursts.values():
        if value_list[len_-1] > threshold and value_list[len_-1] > value_list[len_-2]:
            bursts[len_-1] = len_-1

    if first_n:
        ind_n = []
        for i in bursts.keys():
            ind_n.append(i)
        value_max = [value_list[i] for i in ind_n]
        first_n_ind = np.array(value_max).argsort()[-first_n:]
        burst_new = {}
        for i in first_n_ind:
            burst_new[ind_n[i]] = bursts[ind_n[i]]
        return burst_new
    return bursts

def plot_burst(ind_burst, figsize, frequency, threshold):
    '''
    Plot a graph showing the burst

    input:
    ind_burst: a dictionnary indicating the burst burst and the burst end
    figsize: the size of figure
    frequency: the numbers of tweets for every period
    threshold: the burst below mean value * threshold is ignored

    output:
    figure
    '''
    ind_burst_plot = collections.OrderedDict(sorted(ind_burst.items()))

    fig, ax = plt.subplots(figsize = figsize)
    plt.plot(frequency)
    plt.plot([np.mean(frequency) * threshold]*len(frequency))
    c = -1
    for i in ind_burst_plot.keys():
        c = c + 1
        max_tmp = frequency[i]
        for j in range(i, len(frequency)):
            if j<= ind_burst_plot[i]:
                if max_tmp<frequency[j]:
                    max_tmp = frequency[j]
                if c % 2 == 0 :
                    plt.scatter(j, frequency[j],c = 'red')
                else:
                    plt.scatter(j, frequency[j],c = 'green')
        ax.annotate(str(c+1),(i, max_tmp*1.02))

    plt.legend(('change trend', 'thresholod', 'burst'))
    plt.xlabel('period sequence')
    plt.ylabel('number of tweets')
    plt.title('Find burst according to tweets frequency.')

def split_by_time(dates_, periode = 3600):
    '''
    split our data by a given periode

    input:
    dates_: the published time of tweets
    period: the length of period, in seconds

    output:
    accumulated_num: accumulated numbers until this period
    num:the numbers of tweets for every period
    '''
    accumulated_num = [0]
    num = []
    begin = 0
    for i in range(1,len(dates_)):
        dur_tmp = dates_[i] - dates_[begin]
        if dur_tmp.total_seconds() > periode:
            accumulated_num.append(i)
            num.append(i-begin)
            begin = i
    return accumulated_num, num

def build_graph(num_words, list_node, burst_i, ind_burst, accumulated_num, terms_stop, thresh):
    '''
    Need to specify the threshold which delete all values smaller than it.
    The idea is to build the TermDocumentMatrix M then get the co-occurence matrix by M.T.dot(M)

    input:
    num_words: the length of the matrix
    list_node: all considered words to build the TermDocumentMatrix
    burst_i: the ith burst in the dictionnary
    ind_burst: a dictionnary indicating the burst burst and the burst end by funtion find_burst
    accumulated_num: accumulated numbers until this period given by function split_by_time
    terms_stop: the list of tokenized tweets from read_bz2 function
    thresh: ignore the value in comatrix which is smaller than thresh * mean

    output:
    sps_acc: TermDocumentMatrix
    comatrix: square matrix
    '''
    burst_start = ind_burst.keys()[burst_i]
    burst_end = ind_burst[burst_start]
    burst_start= accumulated_num[burst_start]
    burst_end = accumulated_num[burst_end+1]
    num_tweet = burst_end - burst_start

    #build word frequency matrix
    rows, cols = num_tweet, num_words
    sps_acc = sps.lil_matrix((rows, cols)) # empty sparse matrix
    for i in range(burst_start, burst_end):
        for j in range(cols):
            if list_node[j] in terms_stop[i]:
                sps_acc[i - burst_start,j] = 1

    comatrix = sps_acc.T.dot(sps_acc)#.todense()
    mean = np.mean(comatrix)
    comatrix[comatrix<thresh*mean] = 0
    comatrix[range(cols),range(cols)] = 0

    return sps_acc, comatrix

def find_original(burst_i, ind_burst, clique_word, whole_tweets, dates_, accumulated_num, terms_stop):
    '''
    find the original tweets according to the clique words

    input:
    burst_i: the ith burst in the dictionnary
    ind_burst: a dictionnary indicating the burst burst and the burst end by funtion find_burst
    clique_word: the words in the clique
    ...

    output:
    max_count: how many words are matched in the clique and the original tweets
    whole_tweets[index]: the original tweet
    dates_[index]: the published time
    '''
    max_count = 0
    max_ind = 0
    for i in range(accumulated_num[list(ind_burst.keys())[burst_i]],  accumulated_num[list(ind_burst.values())[burst_i]+1]):
        count_appear = 0
        for j in clique_word:
            if j in terms_stop[i]:
                count_appear = count_appear + 1
            if count_appear == len(clique_word):
                max_ind = i
                break
            if count_appear > max_count:
                max_count = count_appear
                max_ind = i
    index = max_ind

    return  max_count, whole_tweets[index], dates_[index]

def find_event(keyword, ignore, ind_burst, list_word, accumulated_num, terms_stop, whole_tweets, dates, num_words= 150, thre = 0, gamma = 0.45, numClique = 12, minClique = 6):
    '''
    Detect events for all bursts.
    If a clique is overlap with previous one, it is ignored.

    input:
    keyword: the root word for quasi clique
    ignore: ignore the word for a better quasi clique
    list_word: all existing words, it is prepared for num_words
    num_words: how many words are candidates for quasi clique
    thre: value for function build graph
    gamma: key parameter, how much a new edge is close to the max edge weight
    numClique: max number of nodes in the clique
    minClique: how many nodes the resulting clique should contain. This is to avoid that a clique only contains 2 or 3 words.

    output:
    list_clique: all resulting cliques
    list_date: corresponding published time
    list_whole: corresponding original tweets
    '''
    list_date = []
    list_whole = []
    list_clique = []
    for burst_i in range(len(ind_burst)):
        list_node = list_word[burst_i].most_common(num_words)
        list_node = dict(list_node).keys()
        count_matrix, comatrix = build_graph(len(list_node), list_node, burst_i, ind_burst, accumulated_num, terms_stop, thre)
        df = pd.DataFrame(data = comatrix.todense(), columns = list_node, index = list_node)
        if ignore in list_node:
            df = df.drop([ignore]).drop(ignore,axis = 1)
        G = df.values
        #find the root word
        for i in range(len(list(df.columns))):
            if list(df.columns)[i] == keyword.lower():
                break
        list_i = findQuasiClique(G, i, gamma, numClique)
        clique_word = []
        for i in list_i:
            clique_word.append(list(df.columns)[i])
        if len(clique_word) > minClique:
            if not isOverlop(list_clique, clique_word):
                count, whole, date = find_original(burst_i, ind_burst, clique_word, whole_tweets, dates, accumulated_num, terms_stop)
                if count < 4:
                    continue
                list_date.append(date)
                list_whole.append(whole)
                list_clique.append(clique_word)
    return list_date, list_whole, list_clique

def isOverlop(list_clique, clique_word):
    '''
    To detect if a new resulting clique has a fraction of same words with previous one

    input:
    list_clique:  all resulting cliques until now
    clique_word: new cliques

    output:
    overlap or not
    '''
    for clique in list_clique:
        overlap = 0
        for word in clique_word:
            if word in clique:
                overlap +=1
        if overlap > 0.6 * len(clique_word):
            return True

    return False

def findQuasiClique(G, v, gamma, s):
    '''
    Given an undirected graph, find a quasiclique.

    inputs:
        G: the undirected graph, the type is np.ndata
        V: the vertices set.
        gamma: the parameter of weighted gamma-quasi-clique.
        s: the maximum number of vertice in the clique.
    outputs:
        S: the set of nodes.
    '''
    V = list(np.arange(len(G)))
    S = []
    x, y = np.meshgrid(v, np.arange(len(G)))
    u = np.argmax(G[x,y])
    loop = 7
    while  (isolate(sorted(G[u,:], reverse=True))):
        for i in range(len(G)):
            G[i,u] = 0
            G[u,i] = 0
        x, y = np.meshgrid(v, np.arange(len(G)))
        u = np.argmax(G[x,y])
        loop = loop -1
        if loop == 0:
            break
    if (G[v,u]) <= 0:
        return S
    else:
        S.append(v)
        S.append(u)
        while True:
            if len(S) >= s:
                return S
            else:
                '''find the neighbors for set S'''
                res = list(set(V) - set(S))
                if len(res)>0:
                    x,y = np.meshgrid(S, res)
                    w = G[x,y]
                    w = w.reshape(len(S), len(res))
                    flag = w > 0
                    flag = np.sum(flag, axis=0)
                    flag = flag > 0
                    index = np.int_(flag*np.arange(len(res)))
                    neighbor = np.array(res)[index]
                    if len(neighbor):
                        qs = []
                        for i in neighbor:
                            qs.append(qG(G,S+[i]))
                        index = np.argmax(qs)
                        qG_max = np.array(qs)[index]
                        if qG_max < gamma:
                            return S
                        else:
                            S = S + [neighbor[index]]
                else:
                    return S

def isolate(l):
    '''
    If a node is only connected to few nodes, it is ignored.
    If a node's weight is much bigger than the next node, it is ignored.
    '''
    if l[0] > l[1] * 2:
        return 1
    l = np.asarray(l)
    if len(l[l> 0]) < 0.1*len(l):
        return 1

def qG(G,S):
    '''
    To calculate a ratio used for verifying if the subgraph S of G is a weighted gamma quasi-clique.

    inputs:
        G: a graph, the type is np.ndata.
        S: a subgraph of G, the type is np.ndata.

    output:
        return the ratio if S is not empty, otherwise return 0.
    '''
    if len(S):
        x, y = np.meshgrid(S,S)
        total_w = np.sum(G[x,y])/2
        max_w = np.max(G[x,y])
        return total_w/(max_w*comb(len(S),2,exact=False))
    else:
        return 0


#   coding=utf8
__author__ = 'jacky'

import socket
import select
import sys
import re
import json

from urllib import unquote
from cStringIO import StringIO

PATH_RE = re.compile('/(?P<path>(.*))(\?(?P<params>(.*)))?')
ALLOW_METHOD = ['GET', 'POST']

URL_DISPATCH = ()

def start_server():
    """启动socket监听等初始化动作"""

    #  初始化socket
    server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    server_socket.bind(('127.0.0.1', 8080))
    server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    server_socket.listen(5)

    # 根据平台的区别，绑定不同的处理方法
    if sys.platform != 'linux2':
        poll_method = select_wapper
        remove_socket = remove_select_socket
        # 此处使用set,考虑到去除socket的时候，list的消耗过大
        socket_set = set([server_socket])
    else:
        poll_method = epoll_wapper
        socket_set = select.epoll()
        socket_set.register(server_socket, select.EPOLLIN | select.EPOLLET)
        remove_socket = remove_epoll_socket
    while 1:
        ready_socket_list = poll_method(socket_set)
        if len(ready_socket_list) == 0:
            continue

        # ready_socket = ready_socket_list[0]
        for ready_socket in ready_socket_list:

            if ready_socket == server_socket:
                # 有新的用户请求,加入到set中
                select_accpect_handler(server_socket, socket_set)
                continue
            else:
                handle_request(ready_socket)
                remove_socket(ready_socket, socket_set)

def remove_select_socket(socket_remove, socket_set):
    """移除该socket"""
    socket_set.remove(socket_remove)

def remove_epoll_socket(socket_remove, socket_set):
    """取消某个socket的注册"""
    socket_set.unregister(socket_remove)

def select_accpect_handler(server_socket, socket_set):
    """接受新的socket的方法"""
    _temp_socket = server_socket.accept()[0]
    socket_set.add(_temp_socket)

def epoll_accpect_handler(server_socket, socket_set):
    """接受新的socket的方法"""
    socket_set.register(server_socket.accept(), select.EPOLLIN | select.EPOLLET)

def ananlysis_param(raw_string):
    """解析传入的字符串,返回一个字典"""
    result = {}
    param_list = raw_string.split('&')
    for per_param in param_list:
        param_name, param_value = map(unquote, per_param.split('='))
        result[param_name] = param_value

    return result

def handle_request(client_socket):
    """解析用户请求"""
    global PATH_RE, ALLOW_METHOD
    # 限制用户的url为64K
    buffer_file = StringIO(client_socket.recv(65535))

    # 先读取用户的第一行
    raw_request_line = buffer_file.readline()

    raw_request_list = raw_request_line.split()
    request_command, request_path = raw_request_list[0], raw_request_list[1]

    if not request_command.upper() in ALLOW_METHOD:
        return False

    # 过滤请求路径异常的请求
    _result = PATH_RE.match(request_path)

    # 如果没有匹配成功，说明用户的请求异常
    if not _result:
        return False

    request_path = _result.groupdict()['path']
    request_param = _result.groupdict()['params']

    # 对url中的get参数进行解析
    if request_param:
        request_get_param = ananlysis_param(request_param)
    else:
        request_get_param = {}

    # 逐行的解析用户的剩余头部
    request_head = {}
    while 1:
        _temp_read = buffer_file.readline()
        # 如果这个行不是分隔行
        if _temp_read != '\r\n':
            _temp_head_name, _temp_head_value = _temp_read.split(': ')
            request_head[_temp_head_name] = _temp_head_value
        else:
            # 如果读到了指定的分割行了，跳出循环
            break

    #  如果是POST方法，对body进行解析
    request_post_param = {}
    if request_command.upper() == 'POST':
        # 暂时只考虑简单的一行传入
        requets_post_param = ananlysis_param(buffer_file.readline())

    #  对用户的请求进行分发
    find = False
    for per_dispatch in URL_DISPATCH:
        if re.match(per_dispatch[0], request_path):
            # 分发请求处理
            result = per_dispatch[request_command.upper()](request_head, request_get_param, request_post_param)

            find = True

    if not find:
        # 如果找不到处理该请求的
        result = handler_default_request(request_head, request_get_param, request_post_param)
        client_socket.sendall(result)

    # 发送response
    client_socket.sendall(result)
    # 关闭socket
    client_socket.close()
    return True

def handler_default_request(request_head, get_param, post_param):
    """默认的处理方式"""
    result = '\n'.join(['this is the default handler', json.dumps(get_param), json.dumps(post_param)])
    return result

def handler_jacky_get_request(request_head, request_get_param, request_post_param):
    """处理用户请求，相当于view"""
    result = json.dumps(request_get_param)
    return result

def handler_jacky_post_request(request_head, request_get_param, request_post_param):
    """处理用户请求,相当于view"""
    result = json.dumps(request_post_param)
    return result

def select_wapper(socket_set):
    """select方法的包装，返回结果是一个文件描述符"""
    return select.select(socket_set, [], [], 0.1)[0]

def epoll_wapper(socket_set):
    """epoll方法的包装，返回结果是一个文件描述符"""
    _temp = socket_set.poll(0.5)

    return [fd for fd, event in _temp]

if __name__ == '__main__':
    URL_DISPATCH += (
        ('^jacky/$', {'GET': handler_jacky_get_request, 'POST': handler_jacky_post_request}),
    )

    start_server()


a = [[['Inform', 'Train', 'Choice', 'over 1'], ['Inform', 'Train', 'Choice', '000'], ['Request', 'Train', 'Depart', '?']],
[['Inform', 'Train', 'Depart', 'birmingham new street']],
[['Request', 'Train', 'Day', '?']],
[['Inform', 'Train', 'Day', 'wednesday']],
[['Inform', 'Train', 'Arrive', '20:23'], ['Inform', 'Train', 'Day', 'Wednesday'], ['Inform', 'Train', 'Depart', 'birmingham new street'], ['Inform', 'Train', 'Leave', '17:40']],
[['Inform', 'Train', 'People', '5']],
[['OfferBooked', 'Train', 'Ref', 'A9NHSO9Y']],
[['Inform', 'Hotel', 'Internet', 'yes'], ['Inform', 'Hotel', 'Stars', '4']],
[['Recommend', 'Hotel', 'Name', 'the cambridge belfry']],
[['Inform', 'Hotel', 'Day', 'wednesday'], ['Inform', 'Hotel', 'People', '5'], ['Inform', 'Hotel', 'Stay', '5']],
[['Book', 'Booking', 'Ref', '5NAWGJDC']],
[['thank', 'general', 'none', 'none']],
[['bye', 'general', 'none', 'none']]]

print(a)
for i in range(len(a)):
    print()
    for j in range(len(a[i])):
        one = a[i][j]
        left = "_".join(one[:-1])
        right = one[-1]
        whole = ": ".join([left, right])
        print(whole)


"""
Given a binary tree, determine if it is height-balanced.

For this problem, a height-balanced binary tree is defined as a binary tree in which the depth of the two subtrees of every node never differ by more than 1.

判断是不是 平衡二叉树

"""

# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

"""
S1:
    递归判断.
    分别计算左右子树高度, 如果相差>1  返回 -1(表示不平衡) 否则返回 子树高度
"""
class Solution(object):
    def isBalanced(self, root):
        """
        :type root: TreeNode
        :rtype: bool
        """
        def cal_height(root):
            if not root :
                return 0
            l = cal_height(root.left)
            r = cal_height(root.right)
            if l == -1 or r == -1 or abs(l-r)>1 :
                return -1
            else:
                return max(l,r)+1

        if cal_height(root) < 0 :
            return False
        else:
            return True