from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.metrics import accuracy_score
from xgboost import XGBClassifier
from scipy import stats as st

one_to_left = st.beta(5, 1)
from_zero_positive = st.expon(0, 50)
params = {
    "n_estimators": st.randint(50, 200),
    "max_depth": st.randint(3, 10),
    "learning_rate": st.uniform(0.1, 0.4),
    "colsample_bytree": one_to_left,
    "gamma": st.uniform(5, 10),
    'reg_lambda': st.uniform(0,15),
}

class Xgboost():
    def __init__(self, **extra_params):
        self.parameters_to_tune = {'max_depth': [ 5],
                                   'learning_rate': [ .1, .3],
                                   'n_estimators': [100]}
        self.nb_folds = None
        #self.clf = GridSearchCV(XGBClassifier(n_jobs=-1 ), self.parameters_to_tune,
                               # cv=self.nb_folds, scoring='accuracy')
        self.clf = RandomizedSearchCV(XGBClassifier(n_jobs=-1 ), params, n_iter=extra_params.get('n_iter', 1),
                               cv=self.nb_folds, scoring='accuracy')
        #self.clf = XGBClassifier(**{'colsample_bytree': 0.6164602823634784, 'gamma': 14.911780961746683, 'learning_rate': 0.32262035381523485, 'max_depth': 5, 'n_estimators': 185, 'reg_lambda': 7.4168384689423625})
    def train(self, X_train, y_train, X_val=None, y_val=None):
        print("Start training...")
        self.clf.fit(X_train, y_train, eval_set=[(X_val, y_val)], early_stopping_rounds=50)
        print("Training is over.")
        #print("Best params: {}" .format(self.clf.best_params_))
        #print(self.clf.cv_results_)

    def predict(self, X_test):
        return self.clf.predict(X_test)

    def score(self, X_test, y_test):
        return self.clf.score(X_test, y_test)

    def get_weights(self):
        return self.clf.best_estimator_.coef_


from collections import deque

def solution(queue1, queue2):
    answer = -2

    quesum = (sum(queue1) + sum(queue2))
    target = quesum//2

    que1_sum = sum(queue1)

    que1 = deque(queue1)
    que2 = deque(queue2)
    cnt = 0

   #print(quesum)

    if quesum%2 == 1:

        return -1

    while que1 and que2 and cnt<=4*len(queue1):

        if que1_sum> target:
            temp = que1.popleft()
            que1_sum -= temp
            que2.append(temp)
            cnt += 1

        elif que1_sum< target:
            temp = que2.popleft()
            que1_sum += temp
            que1.append(temp)
            cnt += 1
        else :
            return cnt

    else:
        return -1

    #print

    #return answer

prgm_두 큐 합 같게 만들기.py


from __future__ import absolute_import, division, print_function, unicode_literals

import numpy as np
import os
import six
import tensorflow as tf

from collections import namedtuple
from tensorflow.contrib.learn.python.learn.datasets.mnist import DataSet
from tensorflow.examples.tutorials.mnist import input_data
from tqdm import tqdm

from mnist.reweight import get_model, reweight_random, reweight_autodiff, reweight_hard_mining
from utils.logger import get as get_logger

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'
tf.logging.set_verbosity(tf.logging.ERROR)

flags = tf.flags
flags.DEFINE_float('pos_ratio', 0.995, 'Ratio of positive examples in training')
flags.DEFINE_integer('nrun', 10, 'Number of runs')
flags.DEFINE_integer('ntest', 500, 'Number of testing examples')
flags.DEFINE_integer('ntrain', 5000, 'Number of training examples')
flags.DEFINE_integer('nval', 10, 'Number of validation examples')
flags.DEFINE_bool('verbose', False, 'Whether to print training progress')
flags.DEFINE_bool('tensorboard', True, 'Whether to save training progress')
flags.DEFINE_string('exp', 'baseline', 'Which experiment to run')
FLAGS = tf.flags.FLAGS

log = get_logger()

Config = namedtuple('Config', [
    'reweight', 'lr', 'num_steps', 'random', 'ratio_weighted', 'nval', 'hard_mining', 'bsize'
])

exp_repo = dict()

def RegisterExp(name):
    def _decorator(f):
        exp_repo[name] = f
        return f

    return _decorator

LR = 0.001
NUM_STEPS = 4000

@RegisterExp('baseline')
def baseline_config():
    return Config(
        reweight=False,
        num_steps=NUM_STEPS * 2,
        lr=LR,
        random=False,
        ratio_weighted=False,
        hard_mining=False,
        bsize=100,
        nval=0)

@RegisterExp('hm')
def baseline_config():
    return Config(
        reweight=False,
        num_steps=NUM_STEPS * 2,
        lr=LR,
        random=False,
        ratio_weighted=False,
        hard_mining=True,
        bsize=500,
        nval=0)

@RegisterExp('ratio')
def ratio_config():
    return Config(
        reweight=False,
        num_steps=NUM_STEPS * 2,
        lr=LR,
        random=False,
        ratio_weighted=True,
        hard_mining=False,
        bsize=100,
        nval=0)

@RegisterExp('random')
def dpfish_config():
    return Config(
        reweight=True,
        num_steps=NUM_STEPS * 2,
        lr=LR,
        random=True,
        ratio_weighted=False,
        hard_mining=False,
        bsize=100,
        nval=0)

@RegisterExp('ours')
def ours_config():
    return Config(
        reweight=True,
        num_steps=NUM_STEPS,
        lr=LR,
        random=False,
        ratio_weighted=False,
        hard_mining=False,
        bsize=100,
        nval=FLAGS.nval)

def get_imbalance_dataset(mnist,
                          pos_ratio=0.9,
                          ntrain=5000,
                          nval=10,
                          ntest=500,
                          seed=0,
                          class_0=4,
                          class_1=9):
    rnd = np.random.RandomState(seed)

    # In training, we have 10% 4 and 90% 9.
    # In testing, we have 50% 4 and 50% 9.
    ratio = 1 - pos_ratio
    ratio_test = 0.5

    x_train = mnist.train.images
    y_train = mnist.train.labels
    x_test = mnist.test.images
    y_test = mnist.test.labels
    x_train_0 = x_train[y_train == class_0]
    x_test_0 = x_test[y_test == class_0]

    # First shuffle, negative.
    idx = np.arange(x_train_0.shape[0])
    rnd.shuffle(idx)
    x_train_0 = x_train_0[idx]

    nval_small_neg = int(np.floor(nval * ratio_test))
    ntrain_small_neg = int(np.floor(ntrain * ratio)) - nval_small_neg

    x_val_0 = x_train_0[:nval_small_neg]    # 450 4 in validation.
    x_train_0 = x_train_0[nval_small_neg:nval_small_neg + ntrain_small_neg]    # 500 4 in training.

    if FLAGS.verbose:
        print('Number of train negative classes', ntrain_small_neg)
        print('Number of val negative classes', nval_small_neg)

    idx = np.arange(x_test_0.shape[0])
    rnd.shuffle(idx)
    x_test_0 = x_test_0[:int(np.floor(ntest * ratio_test))]    # 450 4 in testing.

    x_train_1 = x_train[y_train == class_1]
    x_test_1 = x_test[y_test == class_1]

    # First shuffle, positive.
    idx = np.arange(x_train_1.shape[0])
    rnd.shuffle(idx)
    x_train_1 = x_train_1[idx]

    nvalsmall_pos = int(np.floor(nval * (1 - ratio_test)))
    ntrainsmall_pos = int(np.floor(ntrain * (1 - ratio))) - nvalsmall_pos

    x_val_1 = x_train_1[:nvalsmall_pos]    # 50 9 in validation.
    x_train_1 = x_train_1[nvalsmall_pos:nvalsmall_pos + ntrainsmall_pos]    # 4500 9 in training.

    idx = np.arange(x_test_1.shape[0])
    rnd.shuffle(idx)
    x_test_1 = x_test_1[idx]
    x_test_1 = x_test_1[:int(np.floor(ntest * (1 - ratio_test)))]    # 500 9 in testing.

    if FLAGS.verbose:
        print('Number of train positive classes', ntrainsmall_pos)
        print('Number of val positive classes', nvalsmall_pos)

    y_train_subset = np.concatenate([np.zeros([x_train_0.shape[0]]), np.ones([x_train_1.shape[0]])])
    y_val_subset = np.concatenate([np.zeros([x_val_0.shape[0]]), np.ones([x_val_1.shape[0]])])
    y_test_subset = np.concatenate([np.zeros([x_test_0.shape[0]]), np.ones([x_test_1.shape[0]])])

    y_train_pos_subset = np.ones([x_train_1.shape[0]])
    y_train_neg_subset = np.zeros([x_train_0.shape[0]])

    x_train_subset = np.concatenate([x_train_0, x_train_1], axis=0).reshape([-1, 28, 28, 1])
    x_val_subset = np.concatenate([x_val_0, x_val_1], axis=0).reshape([-1, 28, 28, 1])
    x_test_subset = np.concatenate([x_test_0, x_test_1], axis=0).reshape([-1, 28, 28, 1])

    x_train_pos_subset = x_train_1.reshape([-1, 28, 28, 1])
    x_train_neg_subset = x_train_0.reshape([-1, 28, 28, 1])

    # Final shuffle.
    idx = np.arange(x_train_subset.shape[0])
    rnd.shuffle(idx)
    x_train_subset = x_train_subset[idx]
    y_train_subset = y_train_subset[idx]

    idx = np.arange(x_val_subset.shape[0])
    rnd.shuffle(idx)
    x_val_subset = x_val_subset[idx]
    y_val_subset = y_val_subset[idx]

    idx = np.arange(x_test_subset.shape[0])
    rnd.shuffle(idx)
    x_test_subset = x_test_subset[idx]
    y_test_subset = y_test_subset[idx]

    train_set = DataSet(x_train_subset * 255.0, y_train_subset)
    train_pos_set = DataSet(x_train_pos_subset * 255.0, y_train_pos_subset)
    train_neg_set = DataSet(x_train_neg_subset * 255.0, y_train_neg_subset)
    val_set = DataSet(x_val_subset * 255.0, y_val_subset)
    test_set = DataSet(x_test_subset * 255.0, y_test_subset)

    return train_set, val_set, test_set, train_pos_set, train_neg_set

def get_exp_logger(sess, log_folder):
    """Gets a TensorBoard logger."""
    with tf.name_scope('Summary'):
        writer = tf.summary.FileWriter(os.path.join(log_folder), sess.graph)

    class ExperimentLogger():
        def log(self, niter, name, value):
            summary = tf.Summary()
            summary.value.add(tag=name, simple_value=value)
            writer.add_summary(summary, niter)

        def flush(self):
            """Flushes results to disk."""
            writer.flush()

    return ExperimentLogger()

def evaluate(sess, x_, y_, acc_, train_set, test_set):
    # Calculate final results.
    acc_sum = 0.0
    acc_test_sum = 0.0
    train_bsize = 100
    for step in six.moves.xrange(5000 // train_bsize):
        x, y = train_set.next_batch(train_bsize)
        acc = sess.run(acc_, feed_dict={x_: x, y_: y})
        acc_sum += acc

    test_bsize = 100
    for step in six.moves.xrange(500 // test_bsize):
        x_test, y_test = test_set.next_batch(test_bsize)
        acc = sess.run(acc_, feed_dict={x_: x_test, y_: y_test})
        acc_test_sum += acc

    train_acc = acc_sum / float(5000 // train_bsize)
    test_acc = acc_test_sum / float(500 // test_bsize)
    return train_acc, test_acc

def get_acc(logits, y):
    prediction = tf.cast(tf.sigmoid(logits) > 0.5, tf.float32)
    return tf.reduce_mean(tf.cast(tf.equal(prediction, y), tf.float32))

def run(dataset, exp_name, seed, verbose=True):
    pos_ratio = FLAGS.pos_ratio
    ntrain = FLAGS.ntrain
    nval = FLAGS.nval
    ntest = FLAGS.ntest
    folder = os.path.join('ckpt_mnist_imbalance_cnn_p{:d}'.format(int(FLAGS.pos_ratio * 100.0)),
                          exp_name + '_{:d}'.format(seed))
    if not os.path.exists(folder):
        os.makedirs(folder)

    with tf.Graph().as_default(), tf.Session() as sess:
        config = exp_repo[exp_name]()
        bsize = config.bsize
        train_set, val_set, test_set, train_pos_set, train_neg_set = get_imbalance_dataset(
            dataset, pos_ratio=pos_ratio, ntrain=ntrain, nval=config.nval, ntest=ntest, seed=seed)
        # if config.nval == 0:
        #     val_set = BalancedDataSet(train_pos_set, train_neg_set)
        x_ = tf.placeholder(tf.float32, [None, 784], name='x')
        y_ = tf.placeholder(tf.float32, [None], name='y')
        x_val_ = tf.placeholder(tf.float32, [None, 784], name='x_val')
        y_val_ = tf.placeholder(tf.float32, [None], name='y_val')
        ex_wts_ = tf.placeholder(tf.float32, [None], name='ex_wts')
        lr_ = tf.placeholder(tf.float32, [], name='lr')

        # Build training model.
        with tf.name_scope('Train'):
            _, loss_c, logits_c = get_model(
                x_, y_, is_training=True, dtype=tf.float32, w_dict=None, ex_wts=ex_wts_, reuse=None)
            train_op = tf.train.MomentumOptimizer(config.lr, 0.9).minimize(loss_c)

        # Build evaluation model.
        with tf.name_scope('Val'):
            _, loss_eval, logits_eval = get_model(
                x_,
                y_,
                is_training=False,
                dtype=tf.float32,
                w_dict=None,
                ex_wts=ex_wts_,
                reuse=True)
            acc_ = get_acc(logits_eval, y_)

        # Build reweighting model.
        if config.reweight:
            if config.random:
                ex_weights_ = reweight_random(bsize)
            else:
                ex_weights_ = reweight_autodiff(
                    x_,
                    y_,
                    x_val_,
                    y_val_,
                    bsize,
                    min(bsize, nval),
                    eps=0.0,
                    gate_gradients=1)
        else:
            if config.hard_mining:
                ex_weights_ = reweight_hard_mining(x_, y_, positive=True)
            else:
                if config.ratio_weighted:
                    # Weighted by the ratio of each class.
                    ex_weights_ = pos_ratio * (1 - y_) + (1 - pos_ratio) * (y_)
                else:
                    # Weighted by uniform.
                    ex_weights_ = tf.ones([bsize], dtype=tf.float32) / float(bsize)

        if FLAGS.tensorboard:
            exp_logger = get_exp_logger(sess, folder)
        else:
            exp_logger = None

        lr = config.lr
        num_steps = config.num_steps

        acc_sum = 0.0
        acc_test_sum = 0.0
        loss_sum = 0.0
        count = 0
        sess.run(tf.global_variables_initializer())
        for step in six.moves.xrange(num_steps):
            x, y = train_set.next_batch(bsize)
            x_val, y_val = val_set.next_batch(min(bsize, nval))

            # Use 50% learning rate for the second half of training.
            if step > num_steps // 2:
                lr = config.lr / 2.0
            else:
                lr = config.lr

            ex_weights = sess.run(
                ex_weights_, feed_dict={x_: x,
                                        y_: y,
                                        x_val_: x_val,
                                        y_val_: y_val})
            loss, acc, _ = sess.run(
                [loss_c, acc_, train_op],
                feed_dict={
                    x_: x,
                    y_: y,
                    x_val_: x_val,
                    y_val_: y_val,
                    ex_wts_: ex_weights,
                    lr_: lr
                })
            if (step + 1) % 100 == 0:
                train_acc, test_acc = evaluate(sess, x_, y_, acc_, train_set, test_set)
                if verbose:
                    print('Step', step + 1, 'Loss', loss, 'Train acc', train_acc, 'Test acc',
                          test_acc)
                if FLAGS.tensorboard:
                    exp_logger.log(step + 1, 'train acc', train_acc)
                    exp_logger.log(step + 1, 'test acc', test_acc)
                    exp_logger.flush()
                acc_sum = 0.0
                loss_sum = 0.0
                acc_test_sum = 0.0
                count = 0

        # Final evaluation.
        train_acc, test_acc = evaluate(sess, x_, y_, acc_, train_set, test_set)
        if verbose:
            print('Final', 'Train acc', train_acc, 'Test acc', test_acc)
    return train_acc, test_acc

def run_many(dataset, exp_name):
    train_acc_list = []
    test_acc_list = []
    for trial in tqdm(six.moves.xrange(FLAGS.nrun), desc=exp_name):
        train_acc, test_acc = run(
            dataset, exp_name, (trial * 123456789) % 100000, verbose=FLAGS.verbose)
        train_acc_list.append(train_acc)
        test_acc_list.append(test_acc)

    train_acc_list = np.array(train_acc_list)
    test_acc_list = np.array(test_acc_list)
    print(exp_name, 'Train acc {:.3f}% ({:.3f}%)'.format(train_acc_list.mean() * 100.0,
                                                         train_acc_list.std() * 100.0))
    print(exp_name, 'Test acc {:.3f}% ({:.3f}%)'.format(test_acc_list.mean() * 100.0,
                                                        test_acc_list.std() * 100.0))

def main():
    mnist = input_data.read_data_sets("data/mnist", one_hot=False)
    for exp in FLAGS.exp.split(','):
        run_many(mnist, exp)

if __name__ == '__main__':
    main()


import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.select import Select

class a():
    def dropdown(self):
        driver = webdriver.Chrome(executable_path=ChromeDriverManager().install())
        driver.get("https://www.salesforce.com/in/form/signup/freetrial-sales/?d=topnav2-btn-ft")
        d = driver.find_element(By.NAME, "UserTitle")
        dd = Select(d)
        time.sleep(4)

        dd.select_by_visible_text("Marketing / PR Manager")
        time.sleep(4)

        dd.select_by_index(1)
        time.sleep(4)

        dd.select_by_value("CxO_General_Manager_AP")
        time.sleep(4)

b = a()
b.dropdown()


import cv2
import numpy as np

f="1.jpg"
img=cv2.imread(f)
hsv = cv2.cvtColor(img,cv2.COLOR_BGR2HSV)

'''
lower_blue=np.array([100,43,46])
upper_blue=np.array([124,255,255])
mask_blue=cv2.inRange(hsv,lower_blue,upper_blue)
res_blue=cv2.bitwise_and(img,img,mask=mask)
'''

def mask(bgr_img,hsv_img,low,up):
    lower_color = np.array(low)
    upper_color = np.array(up)
    mask_color = cv2.inRange(hsv_img,lower_color,upper_color)
    return cv2.bitwise_and(bgr_img,bgr_img,mask=mask_color)

res_blue = mask(img,hsv,[100,43,46],[124,255,255])
res_green = mask(img,hsv,[35,43,46],[77,255,255])
res_orange = mask(img,hsv,[11,43,46],[25,255,255])
res_red = mask(img,hsv,[0,43,46],[6,255,255])
res_white = mask(img,hsv,[0,0,120],[170,60,255])
res_yellow = mask(img,hsv,[17,165,120],[34,255,255])

res = cv2.add(res_blue,res_green)
res = cv2.add(res,res_orange)
res = cv2.add(res,res_red)
res = cv2.add(res,res_white)
res = cv2.add(res,res_yellow)

cv2.imshow('res_blue',res_blue)
cv2.imshow('res_green',res_green)
cv2.imshow('res_orange',res_orange)
cv2.imshow('res_red',res_red)
cv2.imshow('res_white',res_white)
cv2.imshow('res_yellow',res_yellow)
cv2.imshow('res',res)
cv2.waitKey(0)


from Bio import Phylo
from plotly.graph_objs import *
import plotly
def get_x_positions(tree):
    """Create a mapping of each clade to its horizontal position.

    Dict of {clade: x-coord}
    """
    depths = tree.depths()
    # If there are no branch lengths, assume unit branch lengths
    if not max(depths.values()):
        depths = tree.depths(unit_branch_lengths=True)
    return depths
def get_y_positions(tree):
    """Create a mapping of each clade to its vertical position.

    Dict of {clade: y-coord}.
    Coordinates are negative, and integers for tips.
    """
    maxheight = tree.count_terminals()
    # Rows are defined by the tips
    heights = dict((tip, maxheight - i)
                   for i, tip in enumerate(reversed(tree.get_terminals())))

    # Internal nodes: place at midpoint of children
    def calc_row(clade):
        for subclade in clade:
            if subclade not in heights:
                calc_row(subclade)
        # Closure over heights
        heights[clade] = (heights[clade.clades[0]] +
                          heights[clade.clades[-1]]) / 2.0

    if tree.root.clades:
        calc_row(tree.root)
    return heights

def draw_clade_lines(orientation='horizontal',
                     y_here=0, x_start=0, x_here=0, y_bot=0, y_top=0):
    """Create a line with or without a line collection object.

    Graphical formatting of the lines representing clades in the plot can be
    customized by altering this function.
    """
    if orientation == 'horizontal':
        horizontal_linecollections.append(
            [(x_start,y_here), (x_here, y_here)])
    if orientation == 'vertical':
        vertical_linecollections.append(
            [(x_here, y_bot), (x_here, y_top)])

def draw_clade(clade, x_start, color, lw):
    """Recursively draw a tree, down from the given clade."""
    x_here = x_posns[clade]
    y_here = y_posns[clade]
    # Draw a horizontal line from start to here
    draw_clade_lines(orientation='horizontal',
                     y_here=y_here, x_start=x_start, x_here=x_here)
    label = str(clade)
    labels.append(label)
    if clade.clades:
        # for those point non-terminals.
        # Draw a vertical line connecting all children
        y_top = y_posns[clade.clades[0]]
        y_bot = y_posns[clade.clades[-1]]
        # Only apply widths to horizontal lines, like Archaeopteryx

        draw_clade_lines(orientation='vertical',
                         x_here=x_here, y_bot=y_bot, y_top=y_top)
        # Draw descendents
        for child in clade:
            draw_clade(child, x_here, color, lw)

#tree= Phylo.read('/home/liaoth/project/NR-Pfam/searching_all_species/all_species.newick','newick')
#tree= Phylo.read('/home/liaoth/project/NR-Pfam/searching_all_species/present_AN_similiaty_dis/genusid_level_tree.txt','newick')
#tree= Phylo.read('/home/liaoth/project/NR-Pfam/searching_all_species/present_AN_similiaty_dis/species_level_tree_id.txt','newick')
def main(path):
    tree= Phylo.read(path,'newick')
    global x_posns
    global y_posns
    global horizontal_linecollections
    global vertical_linecollections
    global labels
    x_posns = get_x_positions(tree)
    y_posns = get_y_positions(tree)
    horizontal_linecollections = []
    vertical_linecollections = []
    labels = []

    draw_clade(tree.root,0,'k','')
    datas = []
    labels_draw_text = []
    labels_draw_x = []
    labels_draw_y = []
    labels_backup = labels[::]
    labels = [None if 'INT' in _i else _i for _i in labels]
    labels[0] = None
    for _o,_t in horizontal_linecollections:
        horizontal_draws_x = []
        horizontal_draws_y = []
        horizontal_draws_x.append(_o[0])
        horizontal_draws_y.append(_o[1]-1)

        if labels[horizontal_linecollections.index([_o, _t])]:
            horizontal_draws_x.append(11)
            horizontal_draws_y.append(_t[1]-1)
            labels_draw_text.append(labels_backup[horizontal_linecollections.index([_o, _t])])
            labels_draw_x.append(11)
            labels_draw_y.append(_t[1] - 1)

        else:
            horizontal_draws_x.append(_t[0])
            horizontal_draws_y.append(_t[1]-1)

            labels_draw_text.append(labels_backup[horizontal_linecollections.index([_o, _t])])
            labels_draw_x.append(_t[0])
            labels_draw_y.append(_t[1] - 1)

        trace1 = Scatter(x=horizontal_draws_x,y=horizontal_draws_y,mode='lines',line=Line(color ='#444',width=1),hoverinfo='none',xaxis='x1',yaxis='y1',showlegend=False)
        datas.append(trace1)
    for _o,_t in vertical_linecollections:
        vertical_draws_x = []
        vertical_draws_y = []
        vertical_draws_x.append(_o[0])
        vertical_draws_y.append(_o[1]-1)
        vertical_draws_x.append(_t[0])
        vertical_draws_y.append(_t[1]-1)
        trace1 = Scatter(x=vertical_draws_x, y=vertical_draws_y, mode='lines', line=Line(color ='#444',width=1), hoverinfo='none',xaxis='x1',yaxis='y1',showlegend=False)
        datas.append(trace1)
    return datas,labels,labels_backup,labels_draw_text,labels_draw_x,labels_draw_y
# datas.append(Scatter(x=labels_draw_x,y=labels_draw_y,mode='text',name='',
#                text=[_i for _i in labels if _i],textposition = 'middle left',
#                hoverinfo='none',xaxis='x1',yaxis='y1'))
#

# color = ['rgb(91,155,213)', 'rgb(255,192,0)', 'rgb(112,173,71)', 'rgb(112,48,160)']
# for id_idx,_label in enumerate([_i for _i in labels if _i]):
#     for _idx,path in enumerate(sorted(glob.glob('/home/liaoth/project/NR-Pfam/searching_all_species/*.taxid'))):
#         current_color = color[_idx]
#         if _label in open(path).read():
#             datas.append(Scatter(x=[labels_draw_x[id_idx],labels_draw_x[id_idx]+13], y=[labels_draw_y[id_idx]]*2, mode='line',
#                                  name=path.split('/')[-1].split('.')[0],
#                                  #text=[_i for _i in labels if _i][id_idx], textposition='middle left',
#                                  legendgroup = path.split('/')[-1].split('.')[0],
#                                  hoverinfo='name+y',
#                                  xaxis='x1', yaxis='y1',
#                                  line=Line(color=current_color, width=8)))
#                                  #textfont={'color':current_color}))
#             break
if __name__ == '__main__':
    datas,labels,labels_backup,labels_draw_text,labels_draw_x,labels_draw_y = main('./newick.example')
    layout = dict(yaxis=dict(zeroline=False),width=1400,
                  height=2000)
    fig=Figure(data=datas,layout=layout)
    plotly.offline.plot(fig)


import os

# -------------------------------------------------------------------------
#
# GTK+ modules
#
# -------------------------------------------------------------------------
from gi.repository import Gtk

# -------------------------------------------------------------------------
#
# Gramps modules
#
# -------------------------------------------------------------------------
from gramps.gen.const import GRAMPS_LOCALE as glocale

_ = glocale.translation.gettext
from gramps.gen.config import config
from ._reportdialog import ReportDialog
from ._papermenu import PaperFrame
from ...pluginmanager import GuiPluginManager

PLUGMAN = GuiPluginManager.get_instance()

# -------------------------------------------------------------------------
#
# DocReportDialog class
#
# -------------------------------------------------------------------------
class DocReportDialog(ReportDialog):
    """
    The DocReportDialog base class.  This is a base class for generating
    dialogs for docgen derived reports.
    """

    def __init__(self, dbstate, uistate, option_class, name, trans_name, track=[]):
        """Initialize a dialog to request that the user select options
        for a basic *stand-alone* report."""

        self.style_name = "default"
        self.firstpage_added = False
        self.css = PLUGMAN.process_plugin_data("WEBSTUFF")
        self.dbname = dbstate.db.get_dbname()
        ReportDialog.__init__(
            self, dbstate, uistate, option_class, name, trans_name, track=track
        )

        self.basedocname = None  # keep pylint happy
        self.css_filename = None
        self.doc = None
        self.doc_option_class = None
        self.format = None
        self.html_label = None
        self.paper_label = None

        # Allow for post processing of the format frame, since the
        # show_all task calls events that may reset values

    def init_interface(self):
        ReportDialog.init_interface(self)
        self.doc_type_changed(self.format_menu, preserve_tab=False)

    # ------------------------------------------------------------------------
    #
    # Functions related to selecting/changing the current file format.
    #
    # ------------------------------------------------------------------------
    def make_doc_menu(self, active=None):
        """Build a menu of document types that are appropriate for
        this report.  This menu will be generated based upon the type
        of document (text, draw, graph, etc. - a subclass), whether or
        not the document requires table support, etc."""
        raise NotImplementedError

    def make_document(self):
        """Create a document of the type requested by the user."""
        pstyle = self.paper_frame.get_paper_style()

        if self.doc_options:
            self.doc = self.format(
                self.selected_style, pstyle, self.doc_options, uistate=self.uistate
            )
        else:
            self.doc = self.format(self.selected_style, pstyle, uistate=self.uistate)
        if not self.format_menu.get_active_plugin().get_paper_used():
            # set css filename
            self.doc.set_css_filename(self.css_filename)

        self.options.set_document(self.doc)

    def doc_type_changed(self, obj, preserve_tab=True):
        """This routine is called when the user selects a new file
        format for the report.  It adjusts the various dialog sections
        to reflect the appropriate values for the currently selected
        file format.  For example, an HTML document doesn't need any
        paper size/orientation options, but it does need a css
        file.  Those changes are made here."""
        docgen_plugin = obj.get_active_plugin()
        if docgen_plugin.get_extension():
            self.open_with_app.set_sensitive(True)
        else:
            self.open_with_app.set_sensitive(False)

        # Is this to be a printed report or an electronic report
        # (i.e. a set of web pages)

        old_page = self.notebook.get_current_page()
        if self.firstpage_added:
            self.notebook.remove_page(0)
        if docgen_plugin.get_paper_used():
            self.paper_label = Gtk.Label(label="%s" % _("Paper Options"))
            self.paper_label.set_use_markup(True)
            self.notebook.insert_page(self.paper_frame, self.paper_label, 0)
            self.paper_frame.show_all()
        else:
            self.html_label = Gtk.Label(label="%s" % _("HTML Options"))
            self.html_label.set_use_markup(True)
            self.notebook.insert_page(self.html_grid, self.html_label, 0)
            self.html_grid.show_all()
        if preserve_tab:
            self.notebook.set_current_page(old_page)
        self.firstpage_added = True

        ext_val = docgen_plugin.get_extension()
        if ext_val:
            fname = self.target_fileentry.get_full_path(0)
            (spath, ext) = os.path.splitext(fname)

            fname = spath + "." + ext_val
            self.target_fileentry.set_filename(fname)
            self.target_fileentry.set_sensitive(True)
        else:
            self.target_fileentry.set_filename("")
            self.target_fileentry.set_sensitive(False)

        # Does this report format use styles?
        if self.style_button:
            self.style_button.set_sensitive(docgen_plugin.get_style_support())
            self.style_menu.set_sensitive(docgen_plugin.get_style_support())

        self.basedocname = docgen_plugin.get_basedocname()
        self.doc_option_class = docgen_plugin.get_doc_option_class()
        self.setup_doc_options_frame()
        self.show()

    def setup_format_frame(self):
        """Set up the format frame of the dialog.  This function
        relies on the make_doc_menu() function to do all the hard
        work."""

        self.make_doc_menu(self.options.handler.get_format_name())
        self.format_menu.connect("changed", self.doc_type_changed)
        label = Gtk.Label(label=_("%s:") % _("Output Format"))
        label.set_halign(Gtk.Align.START)
        self.grid.attach(label, 1, self.row, 1, 1)
        self.format_menu.set_hexpand(True)
        self.grid.attach(self.format_menu, 2, self.row, 2, 1)
        self.row += 1

        self.open_with_app = Gtk.CheckButton(label=_("Open with default viewer"))
        self.open_with_app.set_active(config.get("interface.open-with-default-viewer"))
        self.grid.attach(self.open_with_app, 2, self.row, 2, 1)
        self.row += 1

        ext = self.format_menu.get_active_plugin().get_extension()
        if ext is None:
            ext = ""
        else:
            spath = self.get_default_directory()
            if self.options.get_output():
                base = os.path.basename(self.options.get_output())
            else:
                if self.dbname is None:
                    default_name = self.raw_name
                else:
                    default_name = self.dbname + "_" + self.raw_name
                base = "%s.%s" % (default_name, ext)
            spath = os.path.normpath(os.path.join(spath, base))
            self.target_fileentry.set_filename(spath)

    def setup_report_options_frame(self):
        """Set up the html/paper frame of the dialog"""
        self.paper_frame = PaperFrame(
            self.options.handler.get_paper_metric(),
            self.options.handler.get_paper_name(),
            self.options.handler.get_orientation(),
            self.options.handler.get_margins(),
            self.options.handler.get_custom_paper_size(),
        )
        self.setup_html_frame()
        ReportDialog.setup_report_options_frame(self)

    def setup_html_frame(self):
        """Set up the html frame of the dialog.  This sole purpose of
        this function is to grab a pointer for later use in the parse
        html frame function."""

        self.html_grid = Gtk.Grid()
        self.html_grid.set_column_spacing(12)
        self.html_grid.set_row_spacing(6)
        self.html_grid.set_border_width(6)

        label = Gtk.Label(label=_("%s:") % _("CSS file"))
        label.set_halign(Gtk.Align.START)
        self.html_grid.attach(label, 1, 1, 1, 1)

        self.css_combo = Gtk.ComboBoxText()
        self.css_combo.set_hexpand(True)

        self.css_filename = self.options.handler.get_css_filename()
        active_index = 0
        index = 0
        for name, _id in sorted(
            [(self.css[key]["translation"], self.css[key]["id"]) for key in self.css]
        ):
            if self.css[_id]["user"]:
                self.css_combo.append_text(self.css[_id]["translation"])
                # Associate this index number with CSS too:
                self.css[index] = self.css[_id]
                if self.css_filename == self.css[_id]["filename"]:
                    active_index = index
                index += 1

        self.html_grid.attach(self.css_combo, 2, 1, 1, 1)
        self.css_combo.set_active(active_index)

    def parse_format_frame(self):
        """Parse the format frame of the dialog.  Save the user
        selected output format for later use."""
        docgen_plugin = self.format_menu.get_active_plugin()
        self.basedocname = docgen_plugin.get_basedocname()
        self.format = docgen_plugin.get_basedoc()
        format_name = docgen_plugin.get_extension()
        self.options.handler.set_format_name(format_name)

    def parse_html_frame(self):
        """Parse the html frame of the dialog.  Save the user selected
        html template name for later use.  Note that this routine
        retrieves a value whether or not the file entry box is
        displayed on the screen.  The subclass will know whether this
        entry was enabled.  This is for simplicity of programming."""

        active = self.css_combo.get_active()
        if active == -1:  # legal for "no active item" (see 7585, 8189, 9461)
            active = self.style_name
        if self.css:
            self.css_filename = self.css[active]["filename"]
        self.options.handler.set_css_filename(self.css_filename)

    def on_ok_clicked(self, obj):
        """The user is satisfied with the dialog choices.  Validate
        the output file name before doing anything else.  If there is
        a file name, gather the options and create the report."""

        # Is there a filename?  This should also test file permissions, etc.
        if not self.parse_target_frame():
            self.window.run()

        # Preparation
        self.parse_format_frame()
        self.parse_style_frame()
        self.parse_html_frame()

        self.options.handler.set_paper_metric(self.paper_frame.get_paper_metric())
        self.options.handler.set_paper_name(self.paper_frame.get_paper_name())
        self.options.handler.set_orientation(self.paper_frame.get_orientation())
        self.options.handler.set_margins(self.paper_frame.get_paper_margins())
        self.options.handler.set_custom_paper_size(
            self.paper_frame.get_custom_paper_size()
        )

        self.parse_user_options()

        # Create the output document.
        self.make_document()

        self.parse_doc_options()

        # Save options
        self.options.handler.save_options()
        config.set(
            "interface.open-with-default-viewer", self.open_with_app.get_active()
        )


import re
import torch
import numpy as np
import math
import torch.nn as nn
import time
import subprocess
import torch.nn.functional as F
from random import shuffle
from transformers import AutoModel
from model.generator import Beam
from datatool.data import DocField, DocDataset, DocIter

# GRU单元的类
class GRUCell(nn.Module):
    def __init__(self, x_dim, h_dim):
        super(GRUCell, self).__init__()
        self.r = nn.Linear(x_dim + h_dim, h_dim, True)  # 记忆消除门的初始化：输入维度+隐向量的维度，输出隐向量维度
        self.z = nn.Linear(x_dim + h_dim, h_dim, True)  # 隐状态权重门的初始化：输入维度+隐向量的维度，输出隐向量维度

        self.c = nn.Linear(x_dim, h_dim, True)  # 把input变成隐状态
        self.u = nn.Linear(h_dim, h_dim, True)  # 当前隐状态

    def forward(self, x, h):
        rz_input = torch.cat((x, h), -1)
        r = torch.sigmoid(self.r(rz_input))
        z = torch.sigmoid(self.z(rz_input))

        u = torch.tanh(self.c(x) + r * self.u(h))

        new_h = z * h + (1 - z) * u
        return new_h

class SGRU(nn.Module):
    def __init__(self, s_emb, e_emb, sh_dim, eh_dim, label_dim):
        super(SGRU, self).__init__()

        g_dim = sh_dim
        self.s_gru = GRUCell(s_emb + sh_dim + label_dim + eh_dim + g_dim, sh_dim)  # 句子编码、上图层句消息、标签、上图层实体消息、上图层全局隐向量

        self.e_gru = GRUCell(e_emb + sh_dim + label_dim + g_dim, eh_dim)  # 实体编码、上图层句消息、标签、上图层全局隐向量

        self.g_gru = GRUCell(sh_dim + eh_dim, g_dim)  # 上图层句消息、上图层实体消息

    # s_h, e_h, g_h = self.slstm((s_input, e_input), (s_h, e_h), g_h, (smask, wmask))
    def forward(self, it, h, g, mask):
        '''
        :param it: B T 2H
        :param h: B T H
        :param g: B H
        :return:
        '''

        si, ei = it
        sh, eh = h
        smask, wmask = mask

        # update sentence node
        g_expand_s = g.unsqueeze(1).expand_as(sh)
        x = torch.cat((si, g_expand_s), -1)
        new_sh = self.s_gru(x, sh)

        # update entity node
        g_expand_e = g.unsqueeze(1).expand(eh.size(0), eh.size(1), g.size(-1))
        x = torch.cat((ei, g_expand_e), -1)
        new_eh = self.e_gru(x, eh)

        new_sh.masked_fill_((smask == 0).unsqueeze(2), 0)
        new_eh.masked_fill_((wmask == 0).unsqueeze(2), 0)

        # update global
        sh_mean = new_sh.sum(1) / smask.float().sum(1, True)
        eh_mean = new_eh.sum(1) / (wmask.float().sum(1, True) + 1)

        mean = torch.cat((sh_mean, eh_mean), -1)
        new_g = self.g_gru(mean, g)

        return new_sh, new_eh, new_g

class OrderPredictor(nn.Module):
    def __init__(self, dim_mlp, sent_hidden, dropout=0.1):
        super(OrderPredictor, self).__init__()
        self.dp = dropout
        self.d_mlp = dim_mlp
        self.s_hid = sent_hidden

        self.MLP = nn.Sequential(nn.Linear(self.s_hid * 2, self.d_mlp, False), nn.ReLU(), nn.Dropout(self.dp),
                                 nn.Linear(self.d_mlp, self.d_mlp, False), nn.ReLU(), nn.Dropout(self.dp))
        self.PredictWeight = nn.Linear(self.d_mlp, 1, False)  # 正向是1，反向是0

    def forward(self, sentences):
        order_score = self.PredictWeight(self.MLP(sentences))
        order_prob = torch.sigmoid(order_score)
        return order_prob

class GRNGOB(nn.Module):
    def __init__(self, s_emb, e_emb, s_hidden, e_hidden, label_dim, dp=0.1, layer=2, agg='sum'):
        super(GRNGOB, self).__init__()
        self.layer = layer
        self.dp = dp

        self.slstm = SGRU(s_emb, e_emb, s_hidden, e_hidden, label_dim)

        self.s_hid = s_hidden
        self.e_hid = e_hidden

        self.agg = agg

        # sent_neighbour_sent
        self.s_orderemb = nn.Embedding(2, label_dim)
        self.s_edgeemb = nn.Embedding(11, label_dim)

        self.gate0 = nn.Linear(s_hidden + s_hidden + label_dim + label_dim, s_hidden)

        # sent_neighbour_ent
        self.edgeemb = nn.Embedding(4, label_dim)
        self.gate1 = nn.Linear(s_hidden + e_hidden + label_dim, e_hidden + label_dim)
        self.gate2 = nn.Linear(s_hidden + e_hidden + label_dim, s_hidden + label_dim)

    def mean(self, x, m, smooth=0):
        mean = torch.matmul(m, x)
        return mean / (m.sum(2, True) + smooth)

    def sum(self, x, m):
        return torch.matmul(m, x)

    # para, hn = self.encoder(sentences, sents_mask, entity_emb, words_mask, elocs)
    def forward(self, sent, smask, word, wmask, elocs, slocs):
        '''
        :param wmask: Batch* Entities
        :param smask: Batch* Sentences
        :param word: Batch* Entities H
        :param sent: Batch* Sentences H
        :return:
        '''
        batch = sent.size(0)
        snum = smask.size(1)
        wnum = wmask.size(1)

        # batch sent_num word_num
        # B S E
        matrix = sent.new_zeros(batch, snum, wnum).long()  # Batch × Sentences × Entity 实体在句子里的角色

        for ib, eloc in enumerate(elocs):
            for ixw, loc in enumerate(eloc):
                for aftersf_ixs, r in loc:
                    matrix[ib, aftersf_ixs, ixw] = r

        mask_se = (matrix != 0).float()  # Batch × Sentences × Entity 句子、实体邻接矩阵
        mask_se_t = mask_se.transpose(1, 2)

        # B S E H
        label_emb = self.edgeemb(matrix)
        label_emb_t = label_emb.transpose(1, 2)

        # B S S
        # connect two sentence if thay have at least one same entity
        '''
        # *********OLD s2smatrix ***********
        s2smatrix = torch.matmul(mask_se, mask_se_t)
        s2smatrix = s2smatrix != 0
        eye = torch.eye(snum).byte().cuda()
        s2s = smask.new_ones(snum, snum)
        eyemask = (s2s - eye).unsqueeze(0)

        s2smatrix = s2smatrix * eyemask
        s2smatrix = s2smatrix & smask.unsqueeze(1)
        s2smatrix = s2smatrix.float()
        '''

        # *********NEW s2sorder ***********
        target = torch.zeros(batch, snum, snum).float().cuda()
        s2sorder = torch.zeros(batch, snum, snum).float().cuda()
        for ib, sloc in enumerate(slocs):
            for triple in sloc:
                target[ib, triple[0], triple[1]] = 1.
                s2sorder[ib, triple[0], triple[1]] = triple[2]
                s2sorder[ib, triple[1], triple[0]] = 1. - triple[2]
        s2smask = target + target.transpose(-1,-2)

        s_h = torch.zeros_like(sent)
        # print("s_h size:",s_h.size())
        g_h = sent.new_zeros(batch, self.s_hid)
        # print("g_h size:",g_h.size())
        e_h = sent.new_zeros(batch, wnum, self.e_hid)
        # print("e_h size:",e_h.size())

        for i in range(self.layer):
            # 1.aggregation
            # s_neigh_s_h = self.mean(s_h, s2sorder)
            s_neigh_s_h = self.sum(s_h, s2sorder)

            # B S E H
            if self.agg == 'gate':
                # 句对句间
                '''
                s_h_expand = s_h.unsqueeze(2).expand(batch, snum, snum, self.s_hid)
                s_h_expand_t = s_h.unsqueeze(1).expand(batch, snum, snum, self.s_hid)

                s_h_expand_order = torch.cat((s_h_expand_t, order_emb), -1)
                s_h_expand_order_value = torch.cat((s_h_expand_order, value_emb), -1)
                s_s_l = torch.cat((s_h_expand, s_h_expand_order_value), -1)

                gs = torch.sigmoid(self.gate0(s_s_l))
                # s_neigh_s_h = s_h_expand_order_value * gs * s2smask.unsqueeze(3)
                s_neigh_s_h = s_h_expand_t * gs * s2smask.unsqueeze(3)
                s_neigh_s_h = s_neigh_s_h.sum(2)
                '''

                # 句对词
                s_h_expand = s_h.unsqueeze(2).expand(batch, snum, wnum, self.s_hid)
                # print("s_h_expand size:",s_h_expand.size())
                e_h_expand = e_h.unsqueeze(1).expand(batch, snum, wnum, self.e_hid)
                # print("e_h_expand size:",e_h_expand.size())

                # 带上了label信息的实体的隐状态
                e_h_expand_edge = torch.cat((e_h_expand, label_emb), -1)
                # print("e_h_expand_edge:",e_h_expand_edge.size())

                s_e_l = torch.cat((s_h_expand, e_h_expand_edge), -1)
                g = torch.sigmoid(self.gate1(s_e_l))

                s_neigh_e_h = e_h_expand_edge * g * mask_se.unsqueeze(3)
                s_neigh_e_h = s_neigh_e_h.sum(2)

                # 词对句
                s_h_expand = s_h.unsqueeze(1).expand(batch, wnum, snum, self.s_hid)
                s_h_expand_edge = torch.cat((s_h_expand, label_emb_t), -1)

                e_h_expand = e_h.unsqueeze(2).expand(batch, wnum, snum, self.e_hid)

                e_s_l = torch.cat((e_h_expand, s_h_expand_edge), -1)
                g2 = torch.sigmoid(self.gate2(e_s_l))

                e_neigh_s_h = s_h_expand_edge * g2 * mask_se_t.unsqueeze(3)
                e_neigh_s_h = e_neigh_s_h.sum(2)

            s_input = torch.cat((sent, s_neigh_s_h, s_neigh_e_h), -1)
            e_input = torch.cat((word, e_neigh_s_h), -1)

            # 2.update
            s_h, e_h, g_h = self.slstm((s_input, e_input), (s_h, e_h), g_h, (smask, wmask))

        if self.dp > 0:
            s_h = F.dropout(s_h, self.dp, self.training)

        return (s_h, s2smask, target), g_h

class PointerNet(nn.Module):
    def __init__(self, args):
        super(PointerNet, self).__init__()

        self.emb_dp = args.input_drop_ratio
        self.model_dp = args.drop_ratio
        self.d_emb = args.d_emb
        self.sen_enc_type = args.senenc

        self.src_embed = nn.Embedding(args.doc_vocab, 100)  # 把document的词汇编码成embedding

        #self.bert = AutoModel.from_pretrained("bert-base-uncased").to('cuda')
        self.bert=None

        self.sen_enc = nn.LSTM(self.d_emb, args.d_rnn // 2, bidirectional=True,
                               batch_first=True)  # 把document的词汇的embedding编码为句子级别的向量

        self.entityemb = args.entityemb

        self.encoder = GRNGOB(s_emb=args.d_rnn,
                              e_emb=args.d_emb if self.entityemb == 'glove' else args.d_rnn,
                              s_hidden=args.d_rnn,
                              e_hidden=args.ehid, label_dim=args.labeldim,
                              layer=args.gnnl, dp=args.gnndp, agg=args.agg)

        d_mlp = args.d_mlp

        self.order_predictor = OrderPredictor(d_mlp, args.d_rnn)

        self.linears = nn.ModuleList([nn.Linear(args.d_rnn, d_mlp),
                                      nn.Linear(args.d_rnn * 2, d_mlp),
                                      nn.Linear(d_mlp, 1)])
        self.decoder = nn.LSTM(args.d_rnn, args.d_rnn, batch_first=True)
        self.critic = None

    def equip(self, critic):
        self.critic = critic

    def forward(self, src_and_len, tgt_and_len, doc_num, ewords_and_len, elocs, slocs, bert_token, blocs=None):

        document_matrix, para, hcn, key = self.encode(src_and_len, doc_num, ewords_and_len, elocs, slocs,
                                                      bert_token, blocs)
        s_h, s2smask, target = para
        s2smatrix = target+target.transpose(-1, -2)
        batch = s_h.size(0)
        snum = s_h.size(1)
        s_h_expand = s_h.unsqueeze(2).expand(batch, snum, snum, self.encoder.s_hid)
        s_h_expand_t = s_h.unsqueeze(1).expand(batch, snum, snum, self.encoder.s_hid)
        s_nei_s = torch.cat((s_h_expand, s_h_expand_t), dim=-1)
        input = s_nei_s[s2smask == 1]
        order = target[s2smask == 1]

        # order_wo_noise = target[(s2smatrix-s2smask) == 1]
        order_prob = self.order_predictor(input).squeeze()

        loss_func = nn.BCELoss()
        loss = loss_func(order_prob, order)

        if torch.isnan(loss):
            exit('nan')
        return loss

    def rnn_enc(self, src_and_len, doc_num):
        '''
        :param src_and_len: src:batch×batch中的最大句子个数，最大词个数 len是每行句子的词数
        :param doc_num: B, each doc has sentences number
        :return: document matirx:batch，最大句子数（每个文档）,512
        '''
        src, length = src_and_len

        sorted_len, ix = torch.sort(length, descending=True)
        sorted_src = src[ix]

        # bi-rnn must uses pack, else needs mask   speedup
        packed_x = nn.utils.rnn.pack_padded_sequence(sorted_src, sorted_len, batch_first=True)
        x = packed_x.data

        x = self.src_embed(x)

        if self.emb_dp > 0:
            x = F.dropout(x, self.emb_dp, self.training)
        packed_x = nn.utils.rnn.PackedSequence(x, packed_x.batch_sizes)

        # 2 TN H
        states, (hn, _) = self.sen_enc(packed_x)

        # TN T 2H
        allwordstates, _ = nn.utils.rnn.pad_packed_sequence(states, True)

        # TN 2H
        hn = hn.transpose(0, 1).contiguous().view(src.size(0), -1)

        _, recovered_ix = torch.sort(ix, descending=False)
        hn = hn[recovered_ix]
        allwordstates = allwordstates[recovered_ix]

        batch_size = len(doc_num)
        maxdoclen = max(doc_num)
        output = hn.view(batch_size, maxdoclen, -1)

        allwordstates = allwordstates.view(batch_size, -1, hn.size(-1))

        return output, allwordstates

    def bert_enc(self, src, doc_num, ewords, bert_token, blocs):
        batch_size = len(doc_num)
        max_doc_len = max(doc_num)
        max_sent_len = len(bert_token['input_ids'][0]) - 1
        max_word_len = len(ewords[0])
        output = self.bert(**bert_token)
        cls_words = output[0].split([1, max_sent_len], dim=1)
        sentences = cls_words[0].view(batch_size, max_doc_len, -1)
        allwords = cls_words[1].split(max_doc_len, dim=0)
        if ewords.size(1) == 0:
            entities = torch.zeros(batch_size, 0, 768).cuda()
            return sentences, entities

        entities = []
        for batch_index in range(len(ewords)):  # 32
            word_bert_locs = blocs[batch_index]
            batch_word = []
            for word_allsent_loc in word_bert_locs:
                each_word=[]
                for word_eachsent_loc in word_allsent_loc:
                    sent_index, locs = word_eachsent_loc
                    word_each_sent = []
                    for loc in locs:
                        word = []
                        for l in loc:
                            word.append(allwords[batch_index][sent_index, l - 1])
                        word = torch.stack(word).mean(-2)
                        word_each_sent.append(word)
                    word_each_sent = torch.stack(word_each_sent).mean(-2)
                    each_word.append(word_each_sent)
                each_word = torch.stack(each_word).mean(-2)
                batch_word.append(each_word)
            real_len = len(batch_word)
            for i in range(max_word_len-real_len):
                batch_word.append(torch.zeros(768).cuda())
            batch_word = torch.stack(batch_word)
            entities.append(batch_word)
        entities = torch.stack(entities)
        return sentences, entities

    def pre_procces(self, src_and_len, doc_num, ewords_and_len, elocs, slocs, bert_token, blocs=None):
        # get sentence emb and mask
        sentences, words_states = self.rnn_enc(src_and_len, doc_num)
        # sentences, words_states = self.bert_enc(src_and_len[0], doc_num, ewords_and_len[0], bert_token, blocs)

        if self.model_dp > 0:
            sentences = F.dropout(sentences, self.model_dp, self.training)

        batch = sentences.size(0)
        sents_mask = sentences.new_zeros(batch, sentences.size(1)).byte()

        for i in range(batch):
            sents_mask[i, :doc_num[i]] = 1

        sentences.masked_fill_(sents_mask.unsqueeze(2) == 0, 0)

        # get entity emb and mask
        words, _ = ewords_and_len
        # <pad> 1
        words_mask = (words != 1)

        entity_emb = self.src_embed(words)

        if self.emb_dp > 0:
            words_states = F.dropout(entity_emb, self.emb_dp, self.training)
        return sentences, sents_mask, words_states, words_mask

    def enc(self, sentences, sents_mask, words_states, words_mask, elocs, slocs):
        para, hn = self.encoder(sentences, sents_mask, words_states, words_mask, elocs, slocs)
        s_h, s2smask, target = para
        hn = hn.unsqueeze(0)
        cn = torch.zeros_like(hn)
        hcn = (hn, cn)

        keyinput = torch.cat((sentences, s_h), -1)
        key = self.linears[1](keyinput)
        return para, hcn, key

    def encode(self, src_and_len, doc_num, ewords_and_len, elocs, slocs, bert_token, blocs=None):
        sentences, sents_mask, words_states, words_mask = self.pre_procces(src_and_len, doc_num, ewords_and_len, elocs,
                                                                           slocs, bert_token, blocs)

        para, hcn, key = self.enc(sentences, sents_mask, words_states, words_mask, elocs, slocs)
        return sentences, para, hcn, key

    def step(self, prev_y, prev_handc, keys, mask):
        '''
        :param prev_y: (seq_len=B, 1, H)
        :param prev_handc: (1, B, H)
        :return:
        '''
        # 1 B H
        _, (h, c) = self.decoder(prev_y, prev_handc)
        # 1 B H-> B H-> B 1 H
        query = h.squeeze(0).unsqueeze(1)
        query = self.linears[0](query)
        # B N H
        e = torch.tanh(query + keys)
        # B N
        e = self.linears[2](e).squeeze(2)
        '''
        keys = keys.transpose(1, 2)
        e = torch.matmul(query, keys).squeeze(1)
        '''
        mask = (mask != 0)
        e.masked_fill_(mask, -1e9)
        logp = F.log_softmax(e, dim=-1)

        return h, c, logp

    def load_pretrained_emb(self, emb):
        self.src_embed = nn.Embedding.from_pretrained(emb, freeze=False).cuda()
        # self.src_embed = nn.Embedding.from_pretrained(emb, freeze=False)

    def order_predict(self, src_and_len, doc_num, ewords_and_len, elocs, slocs, bert_token, blocs=None, f=None):

        sentences, para, hcn, key = self.encode(src_and_len, doc_num, ewords_and_len, elocs, slocs, bert_token, blocs)
        s_h, s2smask, target = para

        batch = s_h.size(0)
        snum = s_h.size(1)

        s_h_expand = s_h.unsqueeze(2).expand(batch, snum, snum, self.encoder.s_hid)
        s_h_expand_t = s_h.unsqueeze(1).expand(batch, snum, snum, self.encoder.s_hid)
        s_nei_s = torch.cat((s_h_expand, s_h_expand_t), dim=-1)
        input = s_nei_s[s2smask == 1]
        order_prob = self.order_predictor(input)

        total_order = torch.zeros(batch,snum,snum).float().cuda()
        total_order[s2smask==1] = order_prob.squeeze()
        total_order[s2smask==1] = order_prob.squeeze()/(total_order+total_order.transpose(-1,-2))[s2smask==1]
        self.log_slocs(slocs,total_order, f)
        return order_prob, target[s2smask == 1]

    def log_slocs(self, slocs, order_prob, file):

        for ib, sloc in enumerate(slocs):
            for triple in sloc:
                file.write(str(order_prob[ib, triple[0], triple[1]].tolist())+' ')
        file.write('\n')

    def update_slocs(self, slocs, order_prob):
        newslocs = []
        for ib, sloc in enumerate(slocs):
            temp = []
            for triple in sloc:
                temp.append((triple[0], triple[1], order_prob[ib, triple[0], triple[1]]))

            newslocs.append(temp)
        return newslocs

def print_params(model):
    print('total parameters:', sum([np.prod(list(p.size())) for p in model.parameters()]))

# train(args, train_real, dev_real, (DOC, ORDER, GRAPH), checkpoint)
# Namespace(agg='gate', alpha=0.6, batch_size=2, beam_size=64,
# corpus=('aan/train.lower', 'aan/train.eg.20'), d_emb=100, d_mlp=500, d_rnn=500,
# decoding_path='decoding', delay=1, doc_vocab=19279, drop_ratio=0.5, early_stop=5, ehid=150, entityemb='glove',
# eval_every=100, gnndp=0.3, gnnl=3, grad_clip=0.0, initnn='standard', input_drop_ratio=0.5, keep_cpts=1, labeldim=50,
# lang=None, length_ratio=2, load_from=None, load_vocab=False, loss=0, lr=1.0, lrdecay=0, main_path='./',
# max_len=None, maximum_steps=100, mode='train', model='07.21_10.01.', model_path='models', n_heads=2, n_layers=5,
# optimizer='Noam', params='user', patience=0, pool=100, ref=None, reglamb=0, resume=False, save_every=50, seed=1234,
# senenc='bow', share_embed=False, share_vocab=False, smoothing=0.0, test=('aan/test.lower', 'aan/test.eg.20'),
#
# valid=('aan/val.lower', 'aan/val.eg.20'), vocab='aan/vocab.new.100d.lower.pt', vocab_size=40000, warmup=4000, writetrans='decoding/ann_0.5_gdp_0.3_gl3.devorder')
def train(args, train_iter, dev, fields, checkpoint):
    model = PointerNet(args)
    model.cuda()

    if checkpoint is not None:
        model.load_state_dict(checkpoint['model'])
    '''
    #for para in model.bert.embeddings.parameters():
    for para in model.bert.parameters():
        para.requires_grad = False

    for layer in range(9,12):
       for para in model.bert.encoder.layer[layer].parameters():
          para.requires_grad = True
    '''
    #bert_params = list(map(id, model.bert.parameters()))
    #base_params = filter(lambda p: id(p) not in bert_params, model.parameters())

    DOC, ORDER, GRAPH = fields
    # print('1:', DOC.vocab.itos[1])
    model.load_pretrained_emb(DOC.vocab.vectors)

    print_params(model)
    print(model)

    wd = 1e-5
    opt = torch.optim.Adadelta(model.parameters(), lr=args.lr, rho=0.95, weight_decay=wd)
    # opt = torch.optim.Adadelta(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, rho=0.95, weight_decay=wd)
    #opt = torch.optim.Adadelta([
    #    {'params': filter(lambda p: p.requires_grad, model.bert.parameters()), 'lr': 3e-3},
    #    {'params': base_params}], lr=args.lr, rho=0.95, weight_decay=wd)

    best_score = -np.inf
    best_iter = 0

    criterion = nn.NLLLoss(reduction='none')
    model.equip(criterion)

    start = time.time()

    early_stop = args.early_stop

    test_data = DocDataset(path=args.test, text_field=DOC, order_field=ORDER, graph_field=GRAPH)
    test_real = DocIter(test_data, 1, device='cuda', batch_size_fn=None,
                        train=False, repeat=False, shuffle=False, sort=False)

    for epc in range(args.maximum_steps):
        for iters, batch in enumerate(train_iter):
            model.train()

            model.zero_grad()

            t1 = time.time()
            loss_print = model(batch.doc, batch.order, batch.doc_len, batch.e_words, batch.elocs, batch.slocs,
                               batch.token, batch.blocs)
            loss = loss_print
            loss.backward()
            opt.step()

            t2 = time.time()
            if iters % 1 == 0:
                print('epc:{} iter:{} loss:{:.4f} t:{:.3f} lr:{:.1e}'.format(epc, iters + 1, loss, t2 - t1,
                                                                             opt.param_groups[0]['lr']))
        if epc < 5:
            continue

        with torch.no_grad():
            print('valid..............')
            if args.loss:
                score = order_model(args, model, dev, 'loss')
                print('epc:{}, loss:{:.2f} best:{:.2f}\n'.format(epc, score, best_score))
            else:
                score = order_model(args, model, dev)
                print('epc:{}, val acc:{:.4f} best:{:.4f} '.format(epc, score, best_score))

            if score > best_score:
                best_score = score
                best_iter = epc

                print('save best model at epc={}'.format(epc))
                checkpoint = {'model': model.state_dict(),
                              'args': args,
                              'loss': best_score}
                torch.save(checkpoint, '{}/{}.best.pt'.format(args.model_path, args.model))

            if early_stop and (epc - best_iter) >= early_stop:
                print('early stop at epc {}'.format(epc))
                break

    print('\n*******Train Done********{}'.format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
    minutes = (time.time() - start) // 60
    if minutes < 60:
        print('best:{:.2f}, iter:{}, time:{} mins, lr:{:.1e}, '.format(best_score, best_iter, minutes,
                                                                       opt.param_groups[0]['lr']))
    else:
        hours = minutes / 60
        print('best:{:.2f}, iter:{}, time:{:.1f} hours, lr:{:.1e}, '.format(best_score, best_iter, hours,
                                                                            opt.param_groups[0]['lr']))

    checkpoint = torch.load('{}/{}.best.pt'.format(args.model_path, args.model), map_location='cuda')
    model.load_state_dict(checkpoint['model'])

    with torch.no_grad():
        acc = order_model(args, model, test_real, shuflle_times=1)
        print('epc:{}, val Acc:{:.4f} best:{:.4f} '.format(epc, acc, best_score))

def order_model(args, model, dev, dev_metrics=None, shuflle_times=1):
    model.eval()

    if dev_metrics == 'loss':
        total_score = []
        number = 0
        print('loss')
        for iters, dev_batch in enumerate(dev):
            loss = model(dev_batch.doc, dev_batch.order, dev_batch.doc_len, dev_batch.e_words, dev_batch.elocs,
                         dev_batch.slocs, dev_batch.token)
            n = dev_batch.order[0].size(0)
            batch_loss = -loss.item() * n
            total_score.append(batch_loss)
            number += n

        return sum(total_score) / number
    else:
        f = open(args.writetrans, 'w', encoding='utf-8')
        f2 = open('{}.slocs'.format(args.model), 'w', encoding='utf-8')
        truth = []
        predicted = []

        for j, dev_batch in enumerate(dev):
            if len(dev_batch.slocs[0]) < 1:
                print('', file=f)
                continue

            logit, tru = model.order_predict(dev_batch.doc, dev_batch.doc_len, dev_batch.e_words, dev_batch.elocs,
                                             dev_batch.slocs, dev_batch.token, dev_batch.blocs, f=f2)
            truth.append(tru.tolist())
            print('{}'.format(' '.join(map(str, logit.tolist()))), file=f)

            logit[logit > 0.5] = 1.
            logit[logit <= 0.5] = 0.

            predicted.append(logit.tolist())

        # print('{}|||{}'.format(' '.join(map(str, pred)), ' '.join(map(str, truth[-1]))),
        #      file=f)

        import itertools

        from sklearn.metrics import accuracy_score

        # acc = right / total
        acc = accuracy_score(list(itertools.chain.from_iterable(truth)),
                             list(itertools.chain.from_iterable(predicted)))

        print('Acc:', acc)
        f.close()
        f2.close()
        return acc

def valid_model(args, model, dev, field, dev_metrics=None, shuflle_times=1):
    model.eval()

    if dev_metrics == 'loss':
        total_score = []
        number = 0

        for iters, dev_batch in enumerate(dev):
            loss = model(dev_batch.doc, dev_batch.order, dev_batch.doc_len, dev_batch.e_words, dev_batch.elocs,
                         dev_batch.slocs, dev_batch.token)
            n = dev_batch.order[0].size(0)
            batch_loss = -loss.item() * n
            total_score.append(batch_loss)
            number += n

        return sum(total_score) / number
    else:

        if args.beam_size != 1:
            print('beam search with beam', args.beam_size)

        best_acc = []
        for epc in range(shuflle_times):
            truth = []
            predicted = []

            for j, dev_batch in enumerate(dev):
                tru = dev_batch.order[0].view(-1).tolist()
                truth.append(tru)

                if len(tru) == 1:
                    pred = tru
                else:
                    pred = beam_search_pointer(args, model, dev_batch.doc, dev_batch.doc_len, dev_batch.e_words,
                                               dev_batch.elocs, dev_batch.slocs, dev_batch.token)

                predicted.append(pred)

            right, total = 0, 0
            pmr_right = 0
            taus = []
            # pm
            pm_p, pm_r = [], []
            import itertools

            from sklearn.metrics import accuracy_score

            for t, p in zip(truth, predicted):
                if len(p) == 1:
                    right += 1
                    total += 1
                    pmr_right += 1
                    taus.append(1)
                    continue

                eq = np.equal(t, p)
                right += eq.sum()
                total += len(t)

                pmr_right += eq.all()

                # pm
                s_t = set([i for i in itertools.combinations(t, 2)])
                s_p = set([i for i in itertools.combinations(p, 2)])
                pm_p.append(len(s_t.intersection(s_p)) / len(s_p))
                pm_r.append(len(s_t.intersection(s_p)) / len(s_t))

                cn_2 = len(p) * (len(p) - 1) / 2
                pairs = len(s_p) - len(s_p.intersection(s_t))
                tau = 1 - 2 * pairs / cn_2
                taus.append(tau)

            # acc = right / total

            acc = accuracy_score(list(itertools.chain.from_iterable(truth)),
                                 list(itertools.chain.from_iterable(predicted)))

            best_acc.append(acc)

            pmr = pmr_right / len(truth)
            taus = np.mean(taus)

            pm_p = np.mean(pm_p)
            pm_r = np.mean(pm_r)
            pm = 2 * pm_p * pm_r / (pm_p + pm_r)

            print('acc:', acc)

        f.close()
        acc = max(best_acc)
        return acc, pmr, taus, pm

def decode(args, test_real, fields, checkpoint):
    with torch.no_grad():
        model = PointerNet(args)
        model.cuda()

        print('load parameters')
        model.load_state_dict(checkpoint['model'])

        DOC, ORDER = fields
        acc = order_model(args, model, test_real, DOC)
        print('test acc:{}'.format(acc))


# coding: utf-8

import random
import numpy as np
import matplotlib.pyplot as plt
import pickle as pkl
import math

np.set_printoptions(precision=2, linewidth=320,suppress=True)
plt.close('all')

def tracerLettre(let):
    a = -let*np.pi/180;
    coord = np.array([[0, 0]]);
    for i in range(len(a)):
        x = np.array([[1, 0]]);
        rot = np.array([[np.cos(a[i]), -np.sin(a[i])],[ np.sin(a[i]),np.cos(a[i])]])
        xr = x.dot(rot) # application de la rotation
        coord = np.vstack((coord,xr+coord[-1,:]))
    plt.plot(coord[:,0],coord[:,1])
    return

def discretise(X, d):
    intervalle = 360 / d
    M = []
    for xi in X:
        line = (np.floor(xi / intervalle)).astype(int)
        M.append(line)
    return np.array(M)

def groupByLabel( y):
    index = []
    for i in np.unique(y): # pour toutes les classes
        ind, = np.where(y==i)
        index.append(ind)
    return index

def find_label(i, index):
    for label in range(len(index)):
        if i in index[label]:
            return label
    ValueError("Le label n'existe pas dans la base d'appprentissage")

def separeTrainTest(y, pc):
    indTrain = []
    indTest = []
    for i in np.unique(y): # pour toutes les classes
        ind, = np.where(y==i)
        n = len(ind)
        indTrain.append(ind[np.random.permutation(n)][:(int)(np.floor(pc*n))])
        indTest.append(np.setdiff1d(ind, indTrain[-1]))
    return indTrain, indTest

def initGD(X, N):
    S = []
    for xi in X:
        v = np.floor(np.linspace(0,N-.00000001,len(xi)))
        S.append(v.astype(int))
    return np.array(S)

def test_initGD():
    X = np.array([ 1,  9,  8,  8,  8,  8,  8,  9,  3,  4,  5,  6,  6,  6,  7,  7,  8,  9,  0,  0,  0,  1,  1])
    S = [ 0,  0,  0,  0,  0,  0,  1,  1,  1,  1,  1,  1,  2,  2,  2,  2,  2,  3,  3,  3,  3,  3,  3]
    S2 = initGD([X],4)
    #print(S2[0])
    np.testing.assert_almost_equal(S, S2[0])

def learnHMM(allX, allS, N, K, initTo0=False):
    if initTo0:
        A = np.zeros((N,N))
        B = np.zeros((N,K))
        Pi = np.zeros(N)
    else:
        eps = 1e-8
        A = np.ones((N,N))*eps
        B = np.ones((N,K))*eps
        Pi = np.ones(N)*eps
    for i in range(len(allX)):
        xi =allX[i]
        si =allS[i]
        Pi[si[0]] += 1
        for i in range(0,len(si)-1):
            etat = si[i]
            next_etat= si[i+1]
            A[etat][next_etat] += 1
            observation = xi[i]
            B[etat][observation] += 1

    A = A/np.maximum(A.sum(1).reshape(N,1),1) # normalisation
    Pi = Pi/Pi.sum()
    M = []
    for line in B:
        nb = np.sum(line)
        line = line / (1.0*nb)
        M.append(line)

    return Pi, A, np.array(M)

def argMax(vecteur, A,j,debug=False):
    maxi = -1 * float('inf')
    index = None
    for i in range(len(vecteur)):
        transition = math.log(A[i][j])
        if debug:
            print('j : ', j)
            print('log transition ', transition)
            print('A[i] :', A[i])
            print('A[i][j]', A[i][j])
            print('vecteur :', vecteur)
        v = vecteur[i] + math.log(A[i][j])
        if debug:
            print('v : ', v)
            print('maxi :', maxi)
        if v >= maxi:
            maxi = v
            index = i
    if index == None:
        ValueError("Il n'y a pas d'index !")
    return index

def viterbi(X, Pi, A, B,debug=False):
    T = len(X)
    N, K = B.shape
    if debug:
        print('N, K', B.shape)
    # N : etats
    delta = np.zeros((N,T))
    psi = np.zeros((N,T))
    if debug:
        print('Pi taille :', Pi.shape)
    for i in range(N):
        delta[i][0] = math.log(Pi[i]) + math.log(B[i][X[0]])
        psi[i][0] = -1
    if debug:
        print('delta : ',delta)
    for t in range(1, T):
        #A chaque pas de temps
        for j in range(N):
            #Pour chaque etat
            pred = delta[:, t-1]
            if debug:
                print('pred : ', pred)
                print('A ', A)
            i_opt = argMax(pred, A, j,False)
            #on trouve son meilleur predecesseur
            if debug:
                print('i_opt :', i_opt)
            delta[j][t] = (delta[i_opt][t-1] + math.log(A[i_opt][j])) + math.log(B[j][X[t]])
            psi[j][t] = i_opt
    if debug:
        print('delta :', delta)

    S = np.max(delta[:,T-1])
    if debug:
        print('S : ', S)
    #backtring
    s = np.zeros(T)
    s[T-1] = np.argmax(delta[:,t])
    for t in range(T-2,1,-1):
        s[t] = psi[:,t+1][int(s[t+1])]
    if debug:
        print('s : ', s)
    return S, s

def printModel(M):
    print('==================== MODEL ===================')
    print(' pi : ',M[0])
    print('----------------------------------------------')
    print(' A : ', M[1])
    print('----------------------------------------------')
    print(' B : ' ,M[2])
    print('==============================================')

def Baum_Welch(X, Y, N, K):
    #initialisation
    S = initGD(X,N)
    index = groupByLabel(Y)
    converge = False
    cpt = 0
    old = float('inf')
    #print('S : ', S)
    #initialisation
    while not(converge):
        models = []
        L = 0
        for lettre in range(len(np.unique(Y))):
            M = learnHMM(X[index[lettre]], S[index[lettre]], N, K)
            models.append(M)
        #on parcours la base d'apprentissage
        for i in range(len(X)):
            xi = X[i]
            M =  models[find_label(i, index)]
            proba, s = viterbi(xi , M[0], M[1], M[2])
            S[i] = s.astype(int)
            L += proba
        if old == float('inf'):
            pass
        elif (old - L) / (1.0*old) < math.exp(-4):
            converge = True
        old = L
        print('iteration numero : ', cpt)
        print('vraisemblance :' ,old)
        cpt += 1
    return models

def predict(xi, models):
    maxi = -1* float('inf')
    index = None
    for lettre in range(len(models)):
        M = models[lettre]
        proba, s = viterbi(xi.astype(int), M[0], M[1], M[2])
        if proba > maxi:
            maxi = proba
            index = lettre
    return maxi, index

def accuracy(X, Y, models):
    K = len(X)
    Ynum = np.zeros(Y.shape)
    accuracy = 0
    vPred = np.zeros(K)
    for num,char in enumerate(np.unique(Y)):
        Ynum[Y == char] = num
    for i in range(K):
        xi = X[i]
        pred = predict (xi, models)[1]
        vPred[i] = pred
    a =  np.where(vPred != Ynum, 0.,1.).mean()
    print('accuracy : {} %'.format( a * 100))
    return a, vPred, Ynum

def draw(conf,Y):
    plt.figure()
    plt.imshow(conf, interpolation='nearest')
    plt.colorbar()
    plt.xticks(np.arange(26),np.unique(Y))
    plt.yticks(np.arange(26),np.unique(Y))
    plt.xlabel(u'Vérité terrain')
    plt.ylabel(u'Prédiction')
    plt.savefig("mat_conf_lettres.png")

def random_proba(P):
    """ la proba P est deja sous cumsum normalement """
    r = random.random()
    for i in range(len(P)):
        if r <= P[i]:
            return i
    return len(P) - 1

def generateHMM(Pi, A,B, datatype, longueur=10):
    sequence = []
    X = []
    so = random_proba(Pi)
    sequence.append(so)
    xo = random_proba(B[so])
    cpt =0
    previous = so
    while cpt < longueur:
        cpt += 1
        si = random_proba(A[previous])
        xi = random_proba(B[si])
        previous = si
        sequence.append(si)
        X.append(xi)
    return sequence, X

def main():
    with open('lettres.pkl', 'rb') as f:
        data = pkl.load(f, encoding='latin1')
    X = np.array(data.get('letters'))
    Y = np.array(data.get('labels'))
    d = 21
    nCl = 26
    K = 10
    N = 5
    itrain,itest = separeTrainTest(Y,0.8)
    ia = []
    for i in itrain:
        ia += i.tolist()
    it = []
    for i in itest:
        it += i.tolist()

    Xd = discretise(X,K)

    Xtrain = Xd[ia]
    Xtest = Xd[it]

    Ytrain = Y[ia]
    Ytest = Y[it]
    test_initGD()
    S = initGD(X, N)
    b = Xd[Y=='a']
    #print('b : ', b)
    Pi, A, B = learnHMM(Xd[Y=='a'], S[Y=='a'],N,K)
    print('Pi : ', Pi)
    print('A :', A)
    print('B : ', B)
    #le resultat est légèrement différent !
    s_est, p_est = viterbi(Xd[0], Pi, A, B)
    print(s_est)
    print(p_est)
    #print('=============================================================')
    models = Baum_Welch(Xtrain, Ytrain, N, K)
    a,pred, Ynum = accuracy(Xtest,Ytest,models)
    conf = np.zeros((26,26))
    for i in range(len(Ynum)):
        conf[int(pred[i])][int(Ynum[i])] += 1
    draw(conf,Ynum)
    #================================================================
    #          PARTIE GENERATIVE (OPTIONNELLE)
    #================================================================
    n = 3          # nb d'échantillon par classe
    nClred = 5   # nb de classes à considérer
    fig = plt.figure()
    for cl in range(nClred):
        Pic = models[cl][0].cumsum() # calcul des sommes cumulées pour gagner du temps
        Ac = models[cl][1].cumsum(1)
        Bc = models[cl][2].cumsum(1)
        long = np.floor(np.array([len(x) for x in Xd[itrain[cl]]]).mean()) # longueur de seq. à générer = moyenne des observations
        for im in range(n):
            s,x = generateHMM(Pic, Ac, Bc, int(long))
            intervalle = 360./d  # pour passer des états => angles
            newa_continu = np.array([i*intervalle for i in x]) # conv int => double
            sfig = plt.subplot(nClred,n,im+n*cl+1)
            sfig.axes.get_xaxis().set_visible(False)
            sfig.axes.get_yaxis().set_visible(False)
            tracerLettre(newa_continu)
    plt.savefig("lettres_hmm.png")
    plt.show()

main()


#!/usr/bin/env python3
"""
das2020_driver is the driver program for the 2020 Disclosure Avoidance System

"""

import json
import logging
import os
import subprocess
import sys
import time
import re
import gc
import threading

if 'SPARK_HOME' not in os.environ:
    os.environ['SPARK_HOME'] = '/usr/lib/spark'

# Add the location of shared libraries
sys.path.append(os.path.join(os.environ['SPARK_HOME'], 'python'))
sys.path.append(os.path.join(os.environ['SPARK_HOME'], 'python', 'lib', 'py4j-src.zip'))

# Run the DAS and return the DAS object
from das_performance import das_stats
from das_performance import gc_stats
import das_performance.syslog_monitoring
import constants as C
from constants import CC

import das_framework.ctools as ctools
import das_framework.ctools.clogging as clogging
import das_framework.ctools.cspark as cspark
import das_framework.driver
import programs.das_setup
import programs.dashboard as dashboard

MIN_FRAMEWORK_VERSION='1.0.0'
MIN_CTOOLS_VERSION='1.0.0'

def display_mission(name):
    block_width = 43

    def dl(line, fill=' ', ):
        print('==={}==='.format(line.center(block_width, fill)))

    dl("", fill='=')
    dl("DAS RUN AT {}".format(time.asctime()))
    dl(name)
    dl("", fill='=')

def dashboard_heartbeat(*,config):
    print_heartbeat     = config.getboolean(section=C.GUROBI_SECTION,option=CC.PRINT_HEARTBEAT,
                                            fallback=True)
    heartbeat_frequency = config.getint(section=C.GUROBI_SECTION,option=CC.HEARTBEAT_FREQUENCY,
                                        fallback=CC.HEARTBEAT_FREQUENCY_DEFAULT)
    print_heartbeat_frequency = config.getint(section=C.GUROBI_SECTION,option=CC.PRINT_HEARTBEAT_FREQUENCY,
                                              fallback=CC.HEARTBEAT_FREQUENCY_DEFAULT)
    last_print_time = 0
    while True:
        if print_heartbeat:
            if last_print_time + print_heartbeat_frequency > time.time():
                print("{} pid={} dashboard_heartbeat".format(time.asctime(),os.getpid()))
                last_print_time = time.time()
        dashboard.heartbeat()
        time.sleep( min(heartbeat_frequency, print_heartbeat_frequency ))

if __name__=="__main__":
    """Driver program to run the DAS and then upload the statistics"""

    if das_framework.__version__ < MIN_FRAMEWORK_VERSION:
        raise RuntimeError("das_framework is out of date; please update to version {}".format(MIN_FRAMEWORK_VERSION))
    if (not hasattr(ctools,'__version__')) or ctools.__version__ < MIN_CTOOLS_VERSION:
        raise RuntimeError("das_framework.ctools is out of date; please update to version {}".format(MIN_FRAMEWORK_VERSION))

    ## OMG. Make the py4j logger not horrible
    ## See https://stackoverflow.com/questions/34248908/how-to-prevent-logging-of-pyspark-answer-received-and-command-to-send-messag
    logging.getLogger("py4j").setLevel(logging.ERROR)

    # these seem to be not the problem:
    # logging.getLogger('pyspark').setLevel(logging.ERROR)
    # logging.getLogger("matplotlib").setLevel(logging.ERROR)

    # Option processing goes first
    # If we make a BOM, it happens here, and we never return
    (args, config) = das_framework.driver.main_setup()

    ###
    ### Validate Configuration File
    ###
    # Look for deprecated variables
    DEPRECATED_CONFIG_VARIABLES = [(C.OUTPUT_FNAME, C.WRITER)]
    for (var, section) in DEPRECATED_CONFIG_VARIABLES:
        if var in config[section]:
            raise RuntimeError(f"config file contains deprecated variable {var} in section [{section}]")

    ###
    ### Set up the environment as necessary
    ###

    applicationId = clogging.applicationId()
    logging.info("applicationId: %s",applicationId)
    os.environ[CC.APPLICATIONID_ENV] = applicationId
    os.environ[CC.PYTHON_VERSION] = f'python{sys.version_info.major}.{sys.version_info.minor}'

    if CC.CLUSTERID_ENV not in os.environ:
        logging.warning("{} environment variable not set; setting to {}".format(CC.CLUSTERID_ENV, CC.CLUSTERID_UNKNOWN))
        os.environ[CC.CLUSTERID_ENV] = CC.CLUSTERID_UNKNOWN

    ###
    ### Set up Gurobi
    ###
    # Add the appropriate Gurobi directory to the path.
    # This must be done *before* 'import gurobi' is executed.
    # Note: we also need to do this on the executors
    if C.GUROBI_PATH in config[C.GUROBI_SECTION]:
        if 'gurobipy' in sys.modules:
            raise RuntimeError("Gurobipy has already been imported. It should only be imported in engine modules.")
        os.environ[CC.PYTHON_VERSION] = f'python{sys.version_info.major}.{sys.version_info.minor}'
        gurobi_path = os.path.expandvars(config[C.GUROBI_SECTION][C.GUROBI_PATH])
        sys.path.insert(0, gurobi_path)

    import gurobipy as gb  # Seems unused but probably needed by bom / make release / switching versions?

    ###
    ### Set up the Mission. It may be already set; if so, use it
    ###

    mission_name = dashboard.get_mission_name()
    display_mission(mission_name)
    dashboard.das_log(mission_name + ' starting', extra={'start':'now()'})

    ###
    ### Create the DAS object
    ###

    das = das_framework.driver.main_make_das(args, config)

    # Have all annotations go to das_log, but no need to print, because annotations print
    das.add_annotation_hook(lambda message: programs.dashboard.das_log(message, verbose=False))

    # Create the sourcecode release and give it the same name as the logfile, except change
    # ".log" to ".release.zip"
    assert das.logfilename.endswith(".log")
    release_zipfilename = re.sub(r".log$", C.RELEASE_SUFFIX, das.logfilename)
    assert not os.path.exists(release_zipfilename)
    cmd = [sys.executable, __file__, '--logfile', '/dev/null', '--make_release', release_zipfilename, args.config]
    logging.info("Creating release: {}".format(" ".join(cmd)))
    subprocess.check_call(cmd)
    assert os.path.exists(release_zipfilename)

    ###
    ### Set up Logging and instrumentation
    ###

    # Make sure Spark log file directories exist
    for option in [C.SPARK_LOCAL_DIR, C.SPARK_EVENTLOG_DIR]:
        try:
            os.makedirs(config[C.SPARK][option], exist_ok=True)
        except KeyError as e:
            pass

    heartbeat_frequency = config.getint(section=C.GUROBI_SECTION,option=CC.HEARTBEAT_FREQUENCY,fallback=CC.HEARTBEAT_FREQUENCY_DEFAULT)
    if heartbeat_frequency>0:
        heartbeat_thread = threading.Thread(target=dashboard_heartbeat, name='heartbeat',
                                            daemon=True, kwargs={'config':config})
        heartbeat_thread.start()

    notification_frequency = config.getint(section=C.GUROBI_SECTION,option=CC.NOTIFICATION_FREQUENCY,fallback=CC.NOTIFICATION_FREQUENCY_DEFAULT)
    stat_dict_list   = []       # list of all the stat dicts that we get
    syslog_watcher   = threading.Thread(target=das_performance.syslog_monitoring.monitor_appId,
                                        daemon=True,
                                        name='syslog_watcher',
                                        kwargs={'appId':applicationId, 'existing':False, 'tail':True, 'stat_dict_list':stat_dict_list,
                                                'update_frequency':notification_frequency })
    syslog_watcher.start()

    ###
    ###
    try:
        if das_framework.driver.strtobool(config[CC.STATS_SECTION][CC.NOTIFY_GC_MASTER]):
            gc.callbacks.append(gc_stats.gc_callback)
            logging.info("Logging GC stats")
    except KeyError as e:
        pass

    ###
    ### Populate the Logs
    ###

    # put the applicationId into the environment and log it
    dashboard.das_log(extra={CC.APPLICATIONID: applicationId}, log_spark=True)
    logging.info(json.dumps({CC.APPLICATIONID: applicationId,
                             CC.START: time.time(),
                             CC.FUNC: ' __main__'}))

    # Dump the spark configuration
    if cspark.spark_running():
        from pyspark.sql import SparkSession
        spark = SparkSession.builder.getOrCreate()
        for (key, val) in spark.sparkContext.getConf().getAll():
            logging.info("{}={}".format(key, val))

    # Save the git commit SHA-1 hash to the config file, so we know on
    # which commit it ran
    #
    # WARNING: It is not guaranteed to get the same code that ran,
    # since files can be modified and not committed at the run time
    # (should add message)

    if config.getboolean(C.WRITER, C.SAVE_GIT_COMMIT, fallback=False):
        curdir = os.getcwd()
        os.chdir(os.path.dirname(os.path.abspath(__file__)))
        commit_point = subprocess.run([C.GIT, C.SHOW], stdout=subprocess.PIPE).stdout.decode(C.UTF_8).split("\n")[0]
        if len(subprocess.run([C.GIT, "status", "--porcelain", "--untracked-files=no"], stdout=subprocess.PIPE).stdout) > 0:
            commit_point = commit_point + " (modifications from commit present at run time)"
        elif len(subprocess.run([C.GIT, "status", "--porcelain"], stdout=subprocess.PIPE).stdout) > 0:
            commit_point = commit_point + " (untracked files present at run time)"
        os.chdir(curdir)
        das.config.set(C.READER, C.GIT_COMMIT, commit_point)

    ###
    ### Run the DAS
    ###

    das_framework.driver.main_run_das(das)

    ###
    ### Start the orderly shutdown
    ###

    logging.info(json.dumps({CC.APPLICATIONID: applicationId,
                             CC.TERMINATED: True,
                             CC.STOP: time.time(),
                             CC.FUNC: ' __main__'}))

    # terminate the DFXML writer
    if das.dfxml_writer.filename:
        das.dfxml_writer.exit()
        das_stats.upload(das.dfxml_writer.filename)

    total_seconds = int(das.running_time())
    minutes = int(total_seconds / 60)
    seconds = total_seconds % 60

    das.annotate("collected {} stats from syslog".format(len(stat_dict_list)))
    if config.getboolean(C.WRITER_SECTION, C.UPLOAD_LOGFILE, fallback=False):
        das.annotate("Uploading Logfile and Analysis Report")
        url = das_stats.run_stats_for_logfile(dfxml=das.dfxml_writer.filename, config=config, stat_dict_list=stat_dict_list)
        das.log_and_print("DAS config: {}".format(args.config))
        das.log_and_print("DAS run stats: {}".format(url))

    if das.output_paths:
        das.log_and_print("\nDAS OUTPUT FILES:")
        for path in das.output_paths:
            das.log_and_print('    '+path)
        das.log_and_print("")

    dashboard.das_log(mission_name + ' finished', extra={'stop': 'now()'})
    das.log_and_print(f"{sys.argv[0]}: Elapsed time: {total_seconds} seconds ({minutes} min, {seconds} seconds)")


import tensorflow as tf
import matplotlib.pyplot as plt
from matplotlib.colors import LinearSegmentedColormap
import numpy as np

k_min = 1
k_max = 15

# Hiperparametri
nb_classes = 2
nb_features = 8
class_trans = {'B': 0, 'M': 1, 0: 'B', 1: 'M'}

# Ucitavanje podataka
filename = 'data/Prostate_Cancer.csv'
all_data = np.genfromtxt(filename, dtype='str', delimiter=',', skip_header=1, usecols=range(1, 10))

# Nasumicno mesanje podataka
nb_samples = all_data.shape[0]
indices = np.random.permutation(nb_samples)
all_data = all_data[indices]

# Konvertovanje podataka u brojeve
for row in all_data:
    row[0] = class_trans[row[0]]
all_data = [[0 if x == 'NA' else float(x) for x in row] for row in all_data]

# Deljenje podataka na skup za treniranje i testiranje
train, test = np.split(all_data, [int(len(all_data)*0.8)])

train_y, train_x = zip(*[(x[0], x[1:].tolist()) for x in train])
test_y, test_x = zip(*[(x[0], x[1:].tolist()) for x in test])

train_x = np.array(train_x)
train_y = np.array(train_y)
test_x = np.array(test_x)
test_y = np.array(test_y)

nb_train = len(train)
nb_test = len(test)

ax_k = []
ax_acc = []

for k in range(k_min, k_max+1):
    tf.reset_default_graph()

    # Ulazni podaci
    X = tf.placeholder(shape=(None, nb_features), dtype=tf.float32)
    Y = tf.placeholder(shape=(None), dtype=tf.int32)

    # Upiti
    Q = tf.placeholder(shape=(nb_features), dtype=tf.float32)

    # Racunanje K minimalnih udaljenosti
    dists = tf.sqrt(tf.reduce_sum(tf.square(tf.subtract(X, Q)), axis=1))
    _, idxs = tf.nn.top_k(-dists, k)
    classes = tf.gather(Y, idxs)
    dists = tf.gather(dists, idxs)
    w = tf.fill([k], 1/k)

    # Odabir klase na osnovu frekvencije u skupu najblizih objekata
    w_col = tf.reshape(w, (k, 1))
    one_hot = tf.one_hot(classes, nb_classes)
    scores = tf.reduce_sum(w_col * one_hot, axis=0)
    hyp = tf.argmax(scores)

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # Trening
        matches = 0
        for i in range(nb_test):
            feed = {X: train_x, Y: train_y, Q: test_x[i]}
            hyp_val = sess.run(hyp, feed_dict = feed)
            act_val = test_y[i]
            match = hyp_val == act_val
            matches += 1 if match else 0
            print('{}/{} Predicted: {} Actual: {} Match: {}'
            .format(i+1, nb_test, class_trans[hyp_val], class_trans[act_val], match))

        print('Examples: {} Matches: {}'.format(nb_test, matches))

        # Racunanje tacnosti modela
        accuracy = matches / nb_test
        print('Accuracy: ', accuracy)

        ax_k.append(k)
        ax_acc.append(accuracy)

# Grafik koji prikazuje zavisnost funkcije troska od stepena polinoma
plt.plot(ax_k, ax_acc)
plt.show()

# ODGOVOR 3C
# Sudeći po grafiku, može se zaključiti da model koji uzima
# u obzir više obeležja ima veću tačnost zato što su objekti
# više razdvojeni u više dimenzija.
# Tvrđenje iz prethodog dela zadatka ostaje da važi
# zato što bi se povećanjem vrednosti parametra K u jednom trenutku
# primetilo opadanje tačnosti modela zato što bi veličina klase bila manja
# od broja susednih objekata koje uzimamo u obzir pri određivanju
# klase objekta iz skupa podataka za testiranje.


import requests
import time
import numpy as np
import matplotlib.pyplot as plt

HOST = "127.0.0.1"
PORT = "5000"
server_url = "http://" + HOST + ":" + PORT
response_times = []
cache_response_times = []
dist=[]

source = "115 Brittany Manor Drive, Amherst, MA, USA"
path_percentage = 425
destinations = ["Brittany Manor Drive, Amherst, MA, USA",
"United States Postal Service, University Drive, Amherst, MA, USA",
"University of Massachusetts Amherst, Amherst, MA, USA",
"Brandywine Apartments, Brandywine, Amherst, MA, USA",
"Sunderland, MA, USA" ]

minimize_elevation_gain = True
for destination in destinations:
    request_json = {
                                                     "source": source,
                                                     "destination": destination,
                                                     "algorithm_id": 0,
                                                     "path_percentage": path_percentage,
                                                     "minimize_elevation_gain": minimize_elevation_gain,
                                                     "transportation_mode": 1
}

    post_time_start = time.time()
    server_response = requests.post(f'{server_url}/getroute', json= request_json)
    post_time_end = time.time()
    print(server_response)
    server_json = server_response.json()
    if not "message" in server_json:
        response_times.append(post_time_end - post_time_start)
        dist.append(server_json["best_path_distance"])

for destination in destinations:
    request_json = {
                                                     "source": source,
                                                     "destination": destination,
                                                     "algorithm_id": 0,
                                                     "path_percentage": path_percentage,
                                                     "minimize_elevation_gain": minimize_elevation_gain,
                                                     "transportation_mode": 1
                                                     }

    post_time_start = time.time()
    server_response = requests.post(f'{server_url}/getroute', json= request_json)
    post_time_end = time.time()
    print(server_response)
    server_json = server_response.json()
    if not "message" in server_json:
        cache_response_times.append(post_time_end - post_time_start)

print(f"response_times:{response_times}")
print(f"cache_response_times:{cache_response_times}")

plt.plot(dist,response_times, label ='1st Time API Hit (No Cache)')
plt.plot(dist,cache_response_times, '-.', label ='2nd Time API Hit (Cached)')

plt.xlabel("Path Distance (in meters)")
plt.ylabel("API Response Time (in seconds)")
plt.legend()
plt.title('Response Time for 1st time and 2nd time API Hit for same configuration')
plt.show()


'''
Given an array A of N integers and a number K. The task is to print the numbers with K frequency.
For each testcase, output the array elements in sorted order which are occurring K times in the array if present, else print -1.
'''
from collections import OrderedDict
if __name__ == '__main__':
    T = int(input())
    for _ in range(T):
        num = list(map(int,input().split()))
        N , K = num[0],num[1]
        A = sorted(list(map(int,input().split())))
        d = OrderedDict()
        for ch in A :
            if ch not in d:d[ch]=1
            else:d[ch]+=1
        l = []
        for k,v in d.items():
            if v==K:l.append(k)
        if l==[]:print(-1)
        else:print(*l)


import csv

import openai
import pandas as pd

# Read Key
T = pd.read_csv('../API_key.csv')
key = T['OpenAI API key']
key = key[0]
openai.api_key = key

def opt3_completion(promt, engine='text-davinci-003', temp=0.2, top_p=1.0, tokens=400, freq_pen=0.0, pres_pen=0.0,
                    stop=['<<END>>']):
    prmt = promt.encode('ASCII', errors='ignore').decode()
    r = openai.Completion.create(
        engine=engine,
        prompt=prmt,
        temperature=temp,
        max_tokens=tokens,
        top_p=top_p,
        frequency_penalty=freq_pen,
        presence_penalty=pres_pen,
        stop=stop)
    text = r['choices'][0]['text'].strip()
    return text

if __name__ == '__main__':
    # Enter metadata
    METADATA = pd.read_csv('/Users/marioacuna/Library/Mobile '
                           'Documents/com~apple~CloudDocs/Project_AcuNez/Studies_December2022'
                           '/Studies_December2022_CheckedMA.csv')

    iid = 0
    responses = []
    for itext in METADATA['summary']:
        iid =iid+ 1
        init_text = "From the following text, infer from where the data were taken (leucocytes?, PBMC?, etc), providing  " \
                    "me with the origin of the samples where the " \
                    "and what kind of patients (AD, PD, MS, etc). Nnothing more, be as concise as possible." \
                    " TEXT: \n <<BLOCK>>"

        text = itext
        #print(text)

        prompt = init_text.replace('<<BLOCK>>', text)
        #prompt = prompt + 'JAX: '
        response = opt3_completion(prompt)
        print(str(iid) + " " + response)
        responses.append(response)
        #conversation.append('JAX: %s' % response)

    with open('output.csv', 'w') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(['iteration', 'output'])
        for i, output in enumerate(responses):
            writer.writerow([i, output])


import sys

import pygame

from bullet import Bullet

def check_keydown_events(event, ai_settings, screen, ship, bullets):
    """response to the keydown"""
    if event.key == pygame.K_UP:
        ship.moving_up = True
    elif event.key == pygame.K_DOWN:
        ship.moving_down = True
    elif event.key == pygame.K_SPACE:
        # build a bullet and join it to the group of bullets
        new_bullet = Bullet(ai_settings, screen, ship)
        bullets.add(new_bullet)

def check_keyup_events(event, ship):
    """response to the keyup"""
    if event.key == pygame.K_UP:
        ship.moving_up = False
    elif event.key == pygame.K_DOWN:
        ship.moving_down = False

def check_events(ai_settings, screen, ship, bullets):
    """response to the keyboard and the mouse"""
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()
        elif event.type == pygame.KEYDOWN:
            check_keydown_events(event, ai_settings, screen, ship, bullets)
        elif event.type == pygame.KEYUP:
            check_keyup_events(event, ship)

def update_screen(ai_settings, screen, ship, bullets):
    """update the screen's image, and change to the new screen"""
    # redraw all the bullets after the aliens and the ship
    for bullet in bullets.sprites():
        bullet.draw_bullet()

    # Redraw the screen each time starts the loop
    screen.fill(ai_settings.bg_color)
    ship.blitme()

    # show the lastest drawing screen
    pygame.display.flip()