"""
Scrapper celery tasks
"""
import requests
from celery import Celery

import config
from libs import advert, page
from logmodule import get_logger

LOGGER = get_logger(__name__)

APP = Celery('tasks', broker=config.RABBIT_URL)

def put_ids_to_queue(ids_list):
    """
    Pushes list of advert ids to celery queue
    """
    LOGGER.debug('pushing %s ads to the queue', len(ids_list))
    for advert_id in ids_list:
        fetch_single_advert.delay(advert_id)

@APP.task(rate_limit='28/m', bind=True, default_retry_delay=60 * 60)
def fetch_single_advert(self, rid):
    """
    Fetching single advert
    """
    try:
        advert.fetch_advert(rid, persist=True)
    except requests.exceptions.HTTPError as exc:
        crawler_banned = exc.response.status_code == 403
        if crawler_banned:
            LOGGER.error('seems like we got banned (403), retrying in 1h')
            self.retry()
        LOGGER.error(exc)
    except AttributeError:
        LOGGER.error('Body parse error on %s', rid)

@APP.task(rate_limit='28/m', default_retry_delay=60 * 60)
def iterate_over_search_results(page_num=1):
    """
    Going through pagination results
    """
    LOGGER.info('fetching page %s', page_num)
    try:
        page_body = page.get_sresults_page(page_num)
    except requests.exceptions.HTTPError as exc:
        if exc.response.status_code == 403:
            # self.retry()
            return

    search_result_adverts = advert.extract_advert_ids_from_search_result_page(page_body)
    put_ids_to_queue(search_result_adverts)

    next_page, pages_count = page.get_sresults_pages_info(page_body)
    if next_page > pages_count:
        return

    iterate_over_search_results.delay(next_page)


import os
import tensorflow as tf

from BaseModel import BaseModel
from MyVocab import MyVocab
import utils

tf.app.flags.DEFINE_string("model", "idcnn", "model: idcnn")
tf.app.flags.DEFINE_string("mode", "train", "mode: train | eval | infer")

FLAGS = tf.app.flags.FLAGS

vocab = MyVocab.load("myvocab.pkl")

def create_hparams(flags):
  """Create training hparams."""
  return tf.contrib.training.HParams(
            is_user_input=False,     # 接收用户输入或者直接从tfrecord中读取数据
            record_file="./data/pku.records",   # tfrecord文件的位置, 在is_user_input=False时, 必须要填写正确的路径
            train_record="./data/pku_train.records",
            valid_record="./data/pku_valid.records",

            layer_mode = "lstm",   # 采用的layer的类型, idcnn | lstm
            vocab_size=vocab.vocab_size,
            tags_size=vocab.tags_size,

            embed_dim=100,

            # lstm 相关参数
            lstm_units=50,      # lstm 中隐藏单元的个数
            lstm_layers = 2,    # lstm 层数
            rnn_output_keep_prob=0.5,

            # idcnn 相关参数
            idcnn_blocks = 3,   # idcnn 中有几个 block
            idcnn_layerpb = 5,  # idcnn 的每个 block 中有几个 layer.

            keep_ratio=0.5,
            l2_ratio = 1.0,     # l2 正则化

            # 学习率相关
            lr = 0.001,
            decay_steps = 1000,
            decay_rate = 0.96,

            num_epochs = 20,
            batch_size = 64,

            traininfo_every = 10,
            evaluate_every = 100,
            checkpoint_every = 500,
            output_dir = os.path.abspath(os.path.join(os.path.curdir, "model_checkout"))
          )

def train():
    with tf.Graph().as_default():
        hparams = create_hparams(FLAGS)
        cws_model = BaseModel(hparams)
        with tf.Session() as session:
            cws_model.train(session)

def infer():
    with tf.Graph().as_default():
        hparams = create_hparams(FLAGS)
        hparams.is_user_input = True
        cws_model = BaseModel(hparams)
        with tf.Session() as session:
            # in_sent = "中华人民共和国中央人民政府今天成立了。"
            while True:
                in_sent = input("\n====== in_sent: ")
                chars = list(utils.strQ2B(in_sent).strip())
                chars_length = len(chars)
                char_ids = [vocab.chr2id.get(chr, vocab.OOV_ID) for chr in chars]

                tags, scores = cws_model.infer(session, [char_ids], [chars_length])
                print("------ score: %s ------" % scores[0])
                for item in zip(chars, tags[0].tolist()):
                    print(item[0], vocab.id2tag[item[1]])

def main(args):
    if FLAGS.mode == "train":
        train()
    elif FLAGS.mode == "infer":
        infer()

if __name__ == "__main__":
    tf.app.run()


import numpy as np
import os
import gzip
import json
import copy
import argparse
import itertools

# For headless
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import seaborn as sns

import sis
import sis_visualizations as visualizations
import lime_helper
from sis import eval_bin_class_model
from python_utils import redirect_stdout, redirect_stderr
from numpy_encoder import NumpyJSONEncoder
from rationale_objects import DNASequence, DNASequenceContainer, Rationale, \
                              SIS_RATIONALE_KEY, \
                              IG_SUFF_RATIONALE_KEY, \
                              IG_TOP_RATIONALE_KEY, \
                              IG_FIXED_RATIONALE_KEY, \
                              LIME_FIXED_RATIONALE_KEY, \
                              PERTURB_FIXED_RATIONALE_KEY
from packages.IntegratedGradients.IntegratedGradients import integrated_gradients

from keras.callbacks import ModelCheckpoint
from keras.models import Sequential
from keras.layers import Dense, Dropout, Conv1D, GlobalMaxPooling1D, Embedding
from keras.optimizers import Adam, Adadelta
from keras.preprocessing import sequence, text

from sklearn.model_selection import train_test_split
from scipy.stats import ranksums, entropy

#####################################
#        GLOBALS/HELPERS
#####################################
BASES = ['A', 'C', 'G', 'T']

NUC_TO_ONEHOT = {'A': np.array([1, 0, 0, 0]).reshape(-1, 1),
                 'C': np.array([0, 1, 0, 0]).reshape(-1, 1),
                 'G': np.array([0, 0, 1, 0]).reshape(-1, 1),
                 'T': np.array([0, 0, 0, 1]).reshape(-1, 1)}

NUC_TO_ONEHOT_WITH_N = copy.deepcopy(NUC_TO_ONEHOT)
NUC_TO_ONEHOT_WITH_N['N'] = np.array([0.25, 0.25, 0.25, 0.25]).reshape(-1, 1)

KNOWN_MOTIFS_MAP_PATH = 'data/motif/known_motifs/map'
KNOWN_MOTIFS_MEME_PATH = 'data/motif/known_motifs/ENCODEmotif'

ENCODE_FAST_MAPPING = {k: v for v, k in enumerate(BASES+['N'])}
ENCODE_FAST_EMBEDDINGS = np.array([[1, 0, 0, 0],
                                   [0, 1, 0, 0],
                                   [0, 0, 1, 0],
                                   [0, 0, 0, 1],
                                   [0.25, 0.25, 0.25, 0.25]])

# Returns numpy array of shape 101 x 4 (where 101 is the length of seq)
def encode_seq(seq):
    return np.hstack((NUC_TO_ONEHOT[c] for c in seq)).T

# Returns numpy array of shape 101 x 4 (where 101 is the length of seq)
# Difference from `encode_seq()` is that this replaces 'N' characters with
#   uniform vectors, whereas encode_seq would throw a key error
def encode_rationale(seq):
    return np.hstack((NUC_TO_ONEHOT_WITH_N[c] for c in seq)).T

# Faster implementation (used in LIME pipeline for better performance)
def encode_rationale_fast(seq):
    base_map = [ENCODE_FAST_MAPPING[c] for c in seq]
    return ENCODE_FAST_EMBEDDINGS[base_map]

def decode_seq(x):
    base_idxs = np.argmax(x, axis=1)
    return [BASES[i] for i in base_idxs]

# Rationale vector is a one-hot array, for each position if element is
#   in rationale or not
def compute_rationale_vector(rationale_idxs, dim=101):
    rv = np.zeros(dim)
    rv[rationale_idxs] = 1
    return rv

# Converts decoded_seq and rationale_idxs into a string containing rationale
# Replaces any non-rationale characters with `nonrationale` (default is '-')
def plaintext_rationale(decoded_seq, rationale_idxs, joiner='',
                         nonrationale='-'):
    rationale_idxs_set = set(rationale_idxs)
    text = [decoded_seq[i] if i in rationale_idxs_set else nonrationale \
                for i in range(len(decoded_seq))]
    return joiner.join(text)

# Truncates a rationale string so "NNNATNGNN" becomes just "ATNG"
# (Removes and leading and trailing N's, preserves internal N's)
def truncate_rationale(rationale, pad_char='N'):
    return rationale.strip(pad_char)

#####################################
#        ARG PARSING
#####################################
def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument('-gpu', type=str, default='1')

    parser.add_argument('-dataset', type=str, required=True)
    parser.add_argument('-thresholdpercentile', type=int, default=90)
    parser.add_argument('-databasedir', type=str,
                         default='data/motif/data')
    parser.add_argument('-outbasedir', type=str,
                         default='rationale_results/motif')
    parser.add_argument('-task', type=str, required=True)
    parser.add_argument('-log', type=str)

    parser.add_argument('--train', dest='train', action='store_true')
    parser.set_defaults(train=False)
    # parser.add_argument('--tunehyperparams', dest='tunehyperparams', action='store_true')
    # parser.set_defaults(tunehyperparams=False)
    parser.add_argument('--runsis', dest='runsis', action='store_true')
    parser.set_defaults(runsis=False)
    parser.add_argument('--runalternatives', dest='runalternatives', action='store_true')
    parser.set_defaults(runalternatives=False)
    parser.add_argument('--comparemethods', dest='comparemethods',
                          action='store_true')
    parser.set_defaults(comparemethods=False)

    parser.add_argument('--v', '--verbose', dest='verbose', action='store_true')
    parser.add_argument('--q', '--quiet', dest='verbose', action='store_false')
    parser.set_defaults(verbose=True)

    args = parser.parse_args()
    return args

#####################################
#        PREPROCESSING
#####################################
def load_tf_data(filepath):
    data = []
    with open(filepath, 'r') as f:
        for line in f:
            parsed_line = line.split()
            seq_id, seq, label = parsed_line
            label = int(label)
            data.append((seq_id, seq, label))
    return data

# input lists contain (seq_id, seq, label) triples from `load_tf_data`
# outputs new `test_data` list ensuring that no elements in
#   `test_data` contain the same seq as any input in `train_data`
def remove_test_duplicates(train_data, test_data, verbose=False):
    train_seqs = set((t[1] for t in train_data))
    filtered_test_data = [t for t in test_data if t[1] not in train_seqs]
    if verbose:
        num_filtered = len(test_data) - len(filtered_test_data)
        print('Removed %d test points found in train set.' % num_filtered)
    return filtered_test_data

# `data` is a list of (seq_id, seq, label) tuples
# Returns: X -- array of same length with (1 x 4 x n) one-hot representation
#                 of each seq
#          y -- 1-D array of labels
#    as (X, y)
def parse_tf_data(data, verbose=False):
    X = []
    y = []
    num_errors = 0
    lines_read = 0
    for seq_id, seq, label in data:
        lines_read += 1
        try:
            encoded_seq = encode_seq(seq)
        except KeyError:  # "N" in the input sequence, throw sequence away
            num_errors += 1
            continue
        X.append(encoded_seq)
        y.append(label)
    X = np.array(X)
    y = np.array(y)
    if verbose:
        print('Read %d lines, %d key errors' % (lines_read, num_errors))
    return (X, y)

def create_val_split(X_train, y_train, test_size=0.125, random_state=42):
    X_train, X_val, y_train, y_val = train_test_split(X_train, y_train,
                                                       test_size=0.125,
                                                       random_state=42)
    return (X_train, X_val, y_train, y_val)

#####################################
#        MODELS
#####################################
# Initialize Keras model for 1layer_128motif CNN architecture
def make_tf_binding_cnn_model(input_shape, filters=128, kernel_size=24,
                               dropout=0.5, optimizer=None):
    model = Sequential()
    model.add(Conv1D(filters, kernel_size,
                      padding='same',
                      input_shape=input_shape,
                      activation='relu'))
    model.add(GlobalMaxPooling1D())
    model.add(Dense(32, activation='relu'))
    model.add(Dropout(dropout))
    model.add(Dense(1, activation='sigmoid'))
    if optimizer is None:
        optimizer = Adadelta()
    model.compile(loss='binary_crossentropy',
                  optimizer=optimizer,
                  metrics=['accuracy'])
    return model

# Only used by hyperparameter tuning
def make_adadelta_optimizer(epsilon=None, rho=0.95):
    optimizer = Adadelta(epsilon=epsilon, rho=rho)
    return optimizer

def make_empty_sequence(dims=(101, 4), fill=0.25):
    empty = np.zeros(dims)
    empty.fill(fill)
    return empty

def make_replacement_embedding(dims=4, fill=0.25):
    vec = np.zeros(dims)
    vec.fill(fill)
    return vec

#####################################
#        EVALUATION
#####################################
# Returns a dictionary of summary statistics on vals
def make_stats_dict(vals):
    res = {}
    res['mean'] = np.mean(vals)
    res['median'] = np.median(vals)
    res['min'] = np.min(vals)
    res['max'] = np.max(vals)
    res['std'] = np.std(vals)
    return res

def eval_tf_binding_model(model, X_train, y_train, X_val, y_val,
                            X_test, y_test, verbose=True):
    train_stats = {}

    train_acc, train_auc = eval_bin_class_model(model, X_train, y_train,
                                                    verbose=False)
    train_loss = model.evaluate(x=X_train, y=y_train, verbose=0)[0]
    if verbose:
        print('')
        print('Performance of best model (lowest val loss) ...')
        print('')
        print('Train Accuracy: %.3f' % train_acc)
        print('Train AUC: %.3f' % train_auc)
        print('Train Loss: %.3f' % train_loss)
        print('')

    val_acc, val_auc = eval_bin_class_model(model, X_val, y_val,
                                            verbose=False)
    val_loss = model.evaluate(x=X_val, y=y_val, verbose=0)[0]
    if verbose:
        print('Val Accuracy: %.3f' % val_acc)
        print('Val AUC: %.3f' % val_auc)
        print('Val Loss: %.3f' % val_loss)
        print('')

    test_acc, test_auc = eval_bin_class_model(model, X_test, y_test,
                                            verbose=False)
    test_loss = model.evaluate(x=X_test, y=y_test, verbose=0)[0]
    if verbose:
        print('Test Accuracy: %.3f' % test_acc)
        print('Test AUC: %.3f' % test_auc)
        print('Test Loss: %.3f' % test_loss)
        print('')

    if not 'train' in train_stats:
        train_stats['train'] = {}
    train_stats['train']['accuracy'] = train_acc
    train_stats['train']['auc'] = train_auc
    train_stats['train']['loss'] = train_loss
    train_stats['train']['size'] = len(y_train)
    if not 'val' in train_stats:
        train_stats['val'] = {}
    train_stats['val']['accuracy'] = val_acc
    train_stats['val']['auc'] = val_auc
    train_stats['val']['loss'] = val_loss
    train_stats['val']['size'] = len(y_val)
    if not 'test' in train_stats:
        train_stats['test'] = {}
    train_stats['test']['accuracy'] = test_acc
    train_stats['test']['auc'] = test_auc
    train_stats['test']['loss'] = test_loss
    train_stats['test']['size'] = len(y_test)

    empty = np.zeros((101, 4))
    all_zeros_pred = float(model.predict(np.array([empty]))[0])
    empty.fill(0.25)
    all_025s_pred = float(model.predict(np.array([empty]))[0])
    y_train_mean = np.mean(y_train)

    train_stats['all_zeros_pred'] = all_zeros_pred
    train_stats['all_025s_pred'] = all_025s_pred
    train_stats['y_train_mean'] = y_train_mean

    if verbose:
        print('Pred. on all zeros: ', all_zeros_pred)
        print('')
        print('Pred on all 0.25s: ', all_025s_pred)
        print('')
        print('y-train mean: ', y_train_mean)

    return train_stats

# Pad motif (n x 4) with `replacement` vectors before and after so total
#   size is (total_len x 4).
# Used for computing KL Divergence of aligned known motif to rationales.
def pad_motif(motif, start_idx, replacement, total_len=101):
    num_left_pad = start_idx
    num_right_pad = total_len - motif.shape[0] - num_left_pad
    left_pad = np.repeat(replacement.reshape(1, -1), num_left_pad, axis=0)
    right_pad = np.repeat(replacement.reshape(1, -1), num_right_pad, axis=0)
    padded_motif = np.vstack((left_pad, motif, right_pad))
    assert(padded_motif.shape[0] == total_len)
    assert(padded_motif.shape[1] == motif.shape[1])
    return padded_motif

# Find optimal alignment of motif in rationale, using sliding window
# rationale_window is same shape as motif
# computes log likelihood of rationale_window under motif
def compute_motif_log_likelihood(motif, rationale_window):
    prod = motif * rationale_window  # element-wise multiplication
    expectations = prod.sum(axis=1)  # sum the 4 values at each position
    log_likelihood = np.log2(expectations).sum()
    return log_likelihood

# Assumes motif is (m x 4) and rationale (n x 4), n >= m
# Returns: starting index motif optimally aligned with rationale
#          and score as (start, score) tuple
def align_motif_rationale(motif, rationale, normalize=True):
    best_align_start = None
    best_align_score = None
    motif_len = motif.shape[0]
    rationale_len = rationale.shape[0]
    for start in range(rationale_len - motif_len + 1):
        rationale_window = rationale[start:start+motif_len, :]
        window_score = compute_motif_log_likelihood(motif, rationale_window)
        if best_align_score is None or window_score > best_align_score:
            best_align_score = window_score
            best_align_start = start
    if normalize:
        best_align_score /= float(motif_len)
    return best_align_start, best_align_score

# Takes padded_motif and rationale as inputs (same shape)
#   and computes KL Divergence of the two dists at each position
# Returns 1D array of same length containing KL values
# Assumes P = rationale and Q = known motif
def kl_div_motif_rationale(padded_motif, rationale):
    assert(padded_motif.shape == rationale.shape)
    kl_vals = []
    # TODO: make this more efficient/vectorize
    for i in range(padded_motif.shape[0]):
        P = rationale[i]
        Q = padded_motif[i]
        kl = entropy(P, qk=Q)
        kl_vals.append(kl)
    return np.array(kl_vals)

# Compute entropy along each position of motif and sum across positions
# Assumes motif has shape (n x 4)
def motif_entropy(motif):
    entropy_vals = []
    for i in range(motif.shape[0]):
        ent = entropy(motif[i])
        entropy_vals.append(ent)
    return np.sum(entropy_vals)

# Levenshtein (edit) distance
def levenshtein_distance(s1, s2):
    # From: https://stackoverflow.com/questions/2460177/edit-distance-in-python
    if len(s1) > len(s2):
        s1, s2 = s2, s1
    distances = range(len(s1) + 1)
    for i2, c2 in enumerate(s2):
        distances_ = [i2+1]
        for i1, c1 in enumerate(s1):
            if c1 == c2:
                distances_.append(distances[i1])
            else:
                distances_.append(1 + min((distances[i1], distances[i1 + 1],
                                           distances_[-1])))
        distances = distances_
    return distances[-1]

# Was used for testing, not used any more
def predict_on_motif(motif, replacement, model):
    start_idx = 101-len(motif) #int((101 - len(motif)) / float(2))
    # Use the most likely base at each position to construct discrete motif
    motif_discrete = np.zeros(motif.shape)
    motif_discrete[np.where(motif == np.max(motif, axis=1, keepdims=True))] = 1
    padded_motif = pad_motif(motif_discrete, start_idx, replacement)
    pred = sis.predict_for_embed_sequence([padded_motif], model)[0]
    print(padded_motif)
    print(pred)
    return pred

#####################################
#        I/O
#####################################
def save_idxs(idxs, outfile):
    with open(outfile, 'w') as f:
        map_data = '\n'.join((str(i) for i in idxs))
        f.write(map_data)

def load_idxs(filepath):
    idxs = []
    with open(filepath, 'r') as f:
        for line in f:
            i = int(line.strip())
            idxs.append(i)
    return idxs

# Rationales is a list of (i, rationale_str) tuples
def save_rationale_texts(rationales, outfile):
    lines = ['%d %s' % (i, r) for i, r in rationales]
    with open(outfile, 'w') as f:
        map_data = '\n'.join(lines)
        f.write(map_data)

# save stats dictionary to json
def save_stats(stats, filepath):
    with open(filepath, 'w') as f:
        json.dump(stats, f, cls=NumpyJSONEncoder)

def load_stats(filepath):
    with open(filepath, 'r') as f:
        stats = json.load(f)
        return stats

def load_motifs_map(path):
    motifs_map = {}
    with open(path, 'r') as f:
        for line in f:
            motif, loc = line.strip().split()
            motifs_map[motif] = loc
    return motifs_map

def parse_meme(filepath, replace_zeros_eps=None):
    res = []
    with open(filepath, 'r') as f:
        for line in f:
            line = line.strip()
            data = line.split('\t')
            if len(data) != 4:  # line does not contain floats, not part of matrix
                continue
            vals = [float(x.strip()) for x in data]
            res.append(vals)
    res = np.array(res, dtype='float32')
    if replace_zeros_eps is not None:
        res[res == 0] = replace_zeros_eps
        res = res / np.linalg.norm(res, axis=1, ord=1, keepdims=True)
    return res

#####################################
#        MAIN
#####################################
def main():
    # Get command line args
    args = parse_args()

    # Configure TF/Keras
    sis.tf_config(cuda_visible_devices=args.gpu)

    # Set params
    VERBOSE = args.verbose
    KERAS_VERBOSE = 2 if VERBOSE else 0  # if verbose, hide progress bar

    TASK = args.task  # `motif_occupancy` or `motif_discovery`
    THRESHOLD_PERCENTILE = args.thresholdpercentile
    DATASET = args.dataset
    DATA_BASE_DIR = args.databasedir
    OUT_BASE_DIR = args.outbasedir

    DATA_DIR = os.path.join(DATA_BASE_DIR, TASK, DATASET)
    OUT_DIR = os.path.join(OUT_BASE_DIR, TASK, DATASET)

    # Make directories to OUT_DIR path if not exists
    if not os.path.isdir(OUT_DIR):
        os.makedirs(OUT_DIR)

    LOG = args.log
    if LOG is not None:
        LOG_PATH = os.path.join(OUT_DIR, LOG)
        redirect_stdout(LOG_PATH)

        err_path = os.path.join(OUT_DIR, 'stderr.txt')
        redirect_stderr(err_path)

    train_data_path = os.path.join(DATA_DIR, 'train.data')
    test_data_path = os.path.join(DATA_DIR, 'test.data')

    train_data = load_tf_data(train_data_path)
    test_data = load_tf_data(test_data_path)
    test_data = remove_test_duplicates(train_data, test_data, verbose=VERBOSE)

    X_train, y_train = parse_tf_data(train_data, verbose=VERBOSE)
    X_test, y_test = parse_tf_data(test_data, verbose=VERBOSE)

    # Take 1/8 of training data as validation data (following paper)
    X_train, X_val, y_train, y_val = create_val_split(X_train, y_train)

    assert(len(X_train) == len(y_train))
    assert(len(X_val) == len(y_val))
    assert(len(X_test) == len(y_test))

    if VERBOSE:
        print('# Train: ', len(X_train))
        print('# Val: ', len(X_val))
        print('# Test: ', len(X_test))
        print('')

    input_shape = X_train[0].shape

    model = None
    model_out_name = '1layer_128motif.h5'
    model_path = os.path.join(OUT_DIR, model_out_name)

    test_predictions = None
    threshold = None
    pos_threshold_f = None

    # if args.tunehyperparams:
    #     tuning_stats = {}
    #     tuning_stats_path = os.path.join(OUT_DIR, 'hyperparam_tuning.json')

    #     DROPOUT = [0.1, 0.5, 0.75]
    #     DELTA = [1e-04, 1e-06, 1e-08]
    #     MOMENT = [0.9, 0.99, 0.999]

    #     best_val_loss = None
    #     best_params = None

    #     for dropout, delta, moment in itertools.product(DROPOUT, DELTA, MOMENT):
    #         dropout_str = '%.2f' % dropout
    #         delta_str = '%.03e' % delta
    #         moment_str = '%.3f' % moment
    #         if VERBOSE:
    #             print('Dropout = %s, Delta = %s, Moment = %s' % \
    #                       (dropout_str, delta_str, moment_str))

    #         optimizer = make_adadelta_optimizer(epsilon=delta, rho=moment)
    #         model = make_tf_binding_cnn_model(input_shape, dropout=dropout,
    #                                           optimizer=optimizer)
    #         if VERBOSE:
    #             print(model.summary())

    #         model.fit(X_train, y_train,
    #                   validation_data=(X_val, y_val),
    #                   epochs=5,
    #                   batch_size=100,
    #                   callbacks=[],
    #                   verbose=KERAS_VERBOSE)

    #         model_stats = eval_tf_binding_model(model, X_train, y_train, X_val,
    #                                             y_val, X_test, y_test,
    #                                             verbose=VERBOSE)

    #         val_loss = model_stats['val']['loss']
    #         if best_val_loss is None or val_loss < best_val_loss:
    #             best_val_loss = val_loss
    #             best_params = {'dropout' : dropout_str,
    #                            'delta' : delta_str,
    #                            'moment' : moment_str}

    #         if 'dropout' not in tuning_stats:
    #             tuning_stats['dropout'] = {}
    #         if dropout_str not in tuning_stats['dropout']:
    #             tuning_stats['dropout'][dropout_str] = {}
    #         dropout_dict = tuning_stats['dropout'][dropout_str]
    #         if 'delta' not in dropout_dict:
    #             dropout_dict['delta'] = {}
    #         if delta_str not in dropout_dict['delta']:
    #             dropout_dict['delta'][delta_str] = {}
    #         delta_dict = dropout_dict['delta'][delta_str]
    #         if 'moment' not in delta_dict:
    #             delta_dict['moment'] = {}
    #         if moment_str not in delta_dict['moment']:
    #             delta_dict['moment'][moment_str] = {}
    #         delta_dict['moment'][moment_str] = model_stats

    #     tuning_stats['best_params'] = best_params

    #     # Save train stats dictionary to json
    #     save_stats(tuning_stats, tuning_stats_path)
    #     if VERBOSE:
    #         print('Dumped hyperparam tuning results to json file.\n')

    if args.train:
        train_stats_path = os.path.join(OUT_DIR, 'train_stats.json')

        model = make_tf_binding_cnn_model(input_shape)
        if VERBOSE:
            print(model.summary())
            print('')

        checkpointer = ModelCheckpoint(filepath=model_path,
                                       verbose=int(VERBOSE),
                                       monitor='val_loss',
                                       save_best_only=True,
                                       save_weights_only=True)

        model.fit(X_train, y_train,
                  validation_data=(X_val, y_val),
                  epochs=10,
                  batch_size=128,
                  callbacks=[checkpointer],
                  verbose=KERAS_VERBOSE)

        # Reload the best model from the checkpoint
        model.load_weights(model_path)

        train_stats = eval_tf_binding_model(model, X_train, y_train, X_val,
                                            y_val, X_test, y_test,
                                            verbose=VERBOSE)

        # Plots of predictive distributions
        train_predictions = model.predict(X_train).flatten()
        val_predictions = model.predict(X_val).flatten()
        test_predictions = model.predict(X_test).flatten()

        threshold = float(np.percentile(test_predictions, THRESHOLD_PERCENTILE))

        if VERBOSE:
            print('Computed threshold as %d percentile: %f' % \
                    (THRESHOLD_PERCENTILE, threshold))

        train_stats['threshold'] = threshold

        visualizations.plot_predictive_dist(train_predictions,
                        title='Predictive Distribution on Train Set',
                        savepath=os.path.join(OUT_DIR, 'predictive_train.png'))

        visualizations.plot_predictive_dist(val_predictions,
                        title='Predictive Distribution on Validation Set',
                        savepath=os.path.join(OUT_DIR, 'predictive_val.png'))

        visualizations.plot_predictive_dist(test_predictions,
                        title='Predictive Distribution on Test Set',
                        vertlines=[threshold],
                        savepath=os.path.join(OUT_DIR, 'predictive_test.png'))

        if VERBOSE:
            print('\nSaved plots of predictive distributions.\n')

        # Save train stats dictionary to json
        save_stats(train_stats, train_stats_path)
        if VERBOSE:
            print('Dumped train stats to json file.\n')

    # Load the model if not loaded yet
    if model is None:
        model = make_tf_binding_cnn_model(input_shape)
        model.load_weights(model_path)

    interesting_points_mapfile = os.path.join(OUT_DIR, 'test_map.txt')
    interesting_test_idxs = None

    container_path = os.path.join(OUT_DIR, 'rationales_container')
    container = None

    if args.runsis:
        if VERBOSE:
            print('Starting SIS...')

        replacement_embedding = make_replacement_embedding()
        if VERBOSE:
            print('\nReplacement embedding: ', str(replacement_embedding))

        if test_predictions is None:
            test_predictions = model.predict(X_test).flatten()

        if threshold is None:
            threshold = np.percentile(test_predictions, THRESHOLD_PERCENTILE)
            pos_threshold_f = lambda x: x >= threshold

        interesting_test_idxs = np.where(test_predictions >= threshold)[0]
        # In case any numpy cast/rounding issues, ensure that model predicts
        #   above threshold on all of these:
        interesting_test_idxs = [i for i in interesting_test_idxs if \
            pos_threshold_f(sis.predict_for_embed_sequence(
                            [X_test[i]], model)[0])]

        if VERBOSE:
            print('# interesting test points (pred >= %f): %d' % \
                    (threshold, len(interesting_test_idxs)))

        # Dump the indices of the "interesting" test points to a map file
        save_idxs(interesting_test_idxs, interesting_points_mapfile)

        # Create new container
        container = DNASequenceContainer(threshold=threshold,
            trained_model_path=model_path)

        # Run SIS
        for i in interesting_test_idxs:
            x = X_test[i]
            seq = DNASequence(x=x, i=i,
                                replacement=replacement_embedding,
                                threshold=threshold,
                                threshold_f=pos_threshold_f)
            # Compute all SIS rationales
            seq.run_sis_rationales(model, first_only=False,
                verbose=False)
            seq.set_predictions(
                model,
                [seq.get_rationales(SIS_RATIONALE_KEY)[0]])
            container.add_sequence(seq)

        # Dump container to file
        container.dump_data(container_path)

        if VERBOSE:
            print('\nSaved container with SIS rationales.\n')

    # Load container if not yet loaded
    if container is None and (args.runalternatives or args.comparemethods):
        container = DNASequenceContainer.load_data(container_path)

    if args.runalternatives:
        if VERBOSE:
            print('Running alternative methods...')

        ig = integrated_gradients(model)
        igs_baseline = make_empty_sequence()
        igs_top_baseline = make_empty_sequence(fill=0)
        lime_pipeline = lime_helper.make_pipeline_dna_seq(model,
            encode_rationale_fast)

        # Run alternative methods
        for seq in container.get_sequences():
            seq.run_integrated_gradients_rationale(ig, model, igs_baseline,
                verbose=False)
            seq.run_integrated_gradients_top_rationale(ig, model,
                igs_top_baseline, verbose=False)
            seq.run_integrated_gradients_fixed_length_rationale(ig, model,
                igs_top_baseline, verbose=False)
            seq.run_lime_fixed_length_rationale(lime_pipeline, decode_seq,
                model, verbose=False)
            seq.run_perturb_fixed_length_rationale(model, verbose=False)

        # Dump container to file, with IG rationales
        container.dump_data(container_path)

        if VERBOSE:
            print('\nSaved container with alternative methods rationales.\n')

    stats = {}
    stats_path = os.path.join(OUT_DIR, 'stats.json')

    if args.comparemethods:
        if VERBOSE:
            print('Starting comparison analysis of SIS and IG...')

        threshold = container.get_threshold()
        stats['threshold'] = threshold

        # Compute stats on number of sufficient rationales
        sis_num_suffic_rationales = [len(s.get_rationales(SIS_RATIONALE_KEY)) \
                                            for s in container.get_sequences()]
        stats['num_suffic_rationales'] = make_stats_dict(
            sis_num_suffic_rationales)

        # Use all SIS rationales
        rationale_lens_sis = []
        for s in container.get_sequences():
            for rationale in s.get_rationales(SIS_RATIONALE_KEY):
                rationale_lens_sis.append(len(rationale))
        rationale_lens_ig = [len(s.get_rationales(IG_SUFF_RATIONALE_KEY)[0]) \
                                    for s in container.get_sequences()]
        rationale_lens_top_ig = [len(s.get_rationales(IG_TOP_RATIONALE_KEY)[0]) \
                                    for s in container.get_sequences()]
        rationale_lens_fixed_length_ig = [len(s.get_rationales(IG_FIXED_RATIONALE_KEY)[0]) \
                                    for s in container.get_sequences()]
        rationale_lens_fixed_length_lime = [len(s.get_rationales(LIME_FIXED_RATIONALE_KEY)[0]) \
                                    for s in container.get_sequences()]
        rationale_lens_fixed_length_pert = [len(s.get_rationales(PERTURB_FIXED_RATIONALE_KEY)[0]) \
                                    for s in container.get_sequences()]

        # Create histogram and scatter plots comparing rationale lengths
        lens_hist_data = [(rationale_lens_sis, 10, 'SIS'),
                          (rationale_lens_ig, 25, 'IG'),
                          (rationale_lens_top_ig, 25, 'Top IG')]
        visualizations.plot_hist(lens_hist_data,
                                 title='Rationale Lengths',
                                 xlabel='Rationale Lengths',
                                 ylabel='Normalized Frequency',
                                 normed=True,
                                 savepath=os.path.join(OUT_DIR,
                                     'rationale_lens_hist.png'))

        # Ranksum test comparing the rationale lengths
        rs_stat_ig, rs_p_ig = ranksums(
            rationale_lens_sis, rationale_lens_ig)
        rs_stat_top_ig, rs_p_top_ig = ranksums(
            rationale_lens_sis, rationale_lens_top_ig)

        if not 'rationale_lengths' in stats:
            stats['rationale_lengths'] = {}
        stats['rationale_lengths']['sis'] = make_stats_dict(
            rationale_lens_sis)
        stats['rationale_lengths']['ig'] = make_stats_dict(
            rationale_lens_ig)
        stats['rationale_lengths']['top_ig'] = make_stats_dict(
            rationale_lens_top_ig)
        stats['rationale_lengths']['fixed_length_ig'] = make_stats_dict(
            rationale_lens_fixed_length_ig)
        stats['rationale_lengths']['fixed_length_lime'] = make_stats_dict(
            rationale_lens_fixed_length_lime)
        stats['rationale_lengths']['fixed_length_pert'] = make_stats_dict(
            rationale_lens_fixed_length_pert)
        stats['rationale_lengths']['sis_vs_ig'] = {}
        stats['rationale_lengths']['sis_vs_ig']['ranksum_statistic'] = float(rs_stat_ig)
        stats['rationale_lengths']['sis_vs_ig']['ranksum_pvalue'] = float(rs_p_ig)
        stats['rationale_lengths']['sis_vs_top_ig'] = {}
        stats['rationale_lengths']['sis_vs_top_ig']['ranksum_statistic'] = float(rs_stat_top_ig)
        stats['rationale_lengths']['sis_vs_top_ig']['ranksum_pvalue'] = float(rs_p_top_ig)

        if VERBOSE:
            print('Ranksums test comparing rationale lengths from SIS',
                    'and IG : stat=%f, p=%f' % (rs_stat_ig, rs_p_ig))
            print('Ranksums test comparing rationale lengths from SIS',
                    'and Top IG : stat=%f, p=%f' % \
                    (rs_stat_top_ig, rs_p_top_ig))

        # Compute statistics on Top IG rationale predictions where all
        #  non-rationale features are masked:
        #  -what percentage have predicted scores above threshold?
        #  -predictive distribution statistics (mean, median, etc.)
        top_ig_rationale_batch = []
        for seq in container.get_sequences():
            x_top_ig = seq.get_x_rationale_only(
                [seq.get_rationales(IG_TOP_RATIONALE_KEY)[0]])
            top_ig_rationale_batch.append(x_top_ig)
        top_ig_predictions = sis.predict_for_embed_sequence(
            top_ig_rationale_batch, model)
        frac_top_above = np.sum(
            top_ig_predictions >= threshold) / float(len(top_ig_predictions))
        stats['top_ig'] = {}
        stats['top_ig']['predictive_dist_stats'] = make_stats_dict(
            top_ig_predictions)
        stats['top_ig']['fraction_above_threshold'] = frac_top_above

        # Rationale-only predictions
        # Predictive distribution on rationales only
        stats['rationale_only_predictions'] = {}
        sis_x_rationales_only = []
        ig_x_rationales_only = []
        top_ig_x_rationales_only = []
        ig_fixed_length_x_rationales_only = []
        lime_fixed_length_x_rationales_only = []
        pert_fixed_length_x_rationales_only = []
        for seq in container.get_sequences():
            sis_rationales = seq.get_rationales(SIS_RATIONALE_KEY)
            for r in sis_rationales:
                sis_x_rationales_only.append(seq.get_x_rationale_only([r]))
            ig_x_rationales_only.append(seq.get_x_rationale_only(
                [seq.get_rationales(IG_SUFF_RATIONALE_KEY)[0]]))
            top_ig_x_rationales_only.append(seq.get_x_rationale_only(
                [seq.get_rationales(IG_TOP_RATIONALE_KEY)[0]]))
            ig_fixed_length_x_rationales_only.append(seq.get_x_rationale_only(
                [seq.get_rationales(IG_FIXED_RATIONALE_KEY)[0]]))
            lime_fixed_length_x_rationales_only.append(seq.get_x_rationale_only(
                [seq.get_rationales(LIME_FIXED_RATIONALE_KEY)[0]]))
            pert_fixed_length_x_rationales_only.append(seq.get_x_rationale_only(
                [seq.get_rationales(PERTURB_FIXED_RATIONALE_KEY)[0]]))
        sis_x_rationales_only_preds = sis.predict_for_embed_sequence(
            sis_x_rationales_only, model)
        ig_x_rationales_only_preds = sis.predict_for_embed_sequence(
            ig_x_rationales_only, model)
        top_ig_x_rationales_only_preds = sis.predict_for_embed_sequence(
            top_ig_x_rationales_only, model)
        ig_fixed_length_x_rationales_only_preds = sis.predict_for_embed_sequence(
            ig_fixed_length_x_rationales_only, model)
        lime_fixed_length_x_rationales_only_preds = sis.predict_for_embed_sequence(
            lime_fixed_length_x_rationales_only, model)
        pert_fixed_length_x_rationales_only_preds = sis.predict_for_embed_sequence(
            pert_fixed_length_x_rationales_only, model)
        stats['rationale_only_predictions']['sis'] = make_stats_dict(
            sis_x_rationales_only_preds)
        stats['rationale_only_predictions']['ig'] = make_stats_dict(
            ig_x_rationales_only_preds)
        stats['rationale_only_predictions']['top_ig'] = make_stats_dict(
            top_ig_x_rationales_only_preds)
        stats['rationale_only_predictions']['ig_fixed_length'] = make_stats_dict(
            ig_fixed_length_x_rationales_only_preds)
        stats['rationale_only_predictions']['lime_fixed_length'] = make_stats_dict(
            lime_fixed_length_x_rationales_only_preds)
        stats['rationale_only_predictions']['pert_fixed_length'] = make_stats_dict(
            pert_fixed_length_x_rationales_only_preds)

        # Perturbation analysis (removing individual elements, comparing
        #   effect from removing rationale elements vs. non-rationale elements)
        rationale_diffs_sis = []
        nonrationale_diffs_sis = []
        for seq in container.get_sequences():
            for rationale in seq.get_rationales(SIS_RATIONALE_KEY):
                rationale_diffs, nonrationale_diffs = seq.perturbation_rationale(
                    model, [rationale])
                rationale_diffs_sis.append(rationale_diffs)
                nonrationale_diffs_sis.append(nonrationale_diffs)

        rationale_diffs_sis_vals = np.concatenate(rationale_diffs_sis).ravel()
        nonrationale_diffs_sis_vals = np.concatenate(nonrationale_diffs_sis).ravel()
        rs_stat_pert_sis, rs_p_pert_sis = ranksums(
            rationale_diffs_sis_vals,
            nonrationale_diffs_sis_vals)

        if not 'perturbation' in stats:
            stats['perturbation'] = {}
        if not 'sis' in stats['perturbation']:
            stats['perturbation']['sis'] = {}
        stats['perturbation']['sis']['rationale'] = make_stats_dict(rationale_diffs_sis_vals)
        stats['perturbation']['sis']['nonrationale'] = make_stats_dict(nonrationale_diffs_sis_vals)
        stats['perturbation']['sis']['ranksum_statistic'] = float(rs_stat_pert_sis)
        stats['perturbation']['sis']['ranksum_pvalue'] = float(rs_p_pert_sis)

        if VERBOSE:
            print('Ranksums test comparing perturbation deltas from',
                    'SIS rationale vs non-rationale',
                    'stat=%f, p=%f' % (rs_stat_pert_sis, rs_p_pert_sis))

        pert_hist_sis = [(rationale_diffs_sis_vals, 25, 'Rationale'),
                            (nonrationale_diffs_sis_vals, 25, 'Non-rationale')]
        visualizations.plot_hist(pert_hist_sis,
                                 title='Perturbation Deltas, SIS',
                                 xlabel='original prediction - prediction without word',
                                 ylabel='Normalized Frequency',
                                 normed=True,
                                 savepath=os.path.join(OUT_DIR,
                                     'perturbation_hist_sis.png'))

        rationale_diffs_ig = []
        nonrationale_diffs_ig = []
        for seq in container.get_sequences():
            rationale_diffs, nonrationale_diffs = seq.perturbation_rationale(
                model,
                [seq.get_rationales(IG_SUFF_RATIONALE_KEY)[0]])
            rationale_diffs_ig.append(rationale_diffs)
            nonrationale_diffs_ig.append(nonrationale_diffs)

        rationale_diffs_ig_vals = np.concatenate(rationale_diffs_ig).ravel()
        nonrationale_diffs_ig_vals = np.concatenate(nonrationale_diffs_ig).ravel()
        rs_stat_pert_ig, rs_p_pert_ig = ranksums(
            rationale_diffs_ig_vals,
            nonrationale_diffs_ig_vals)

        if not 'ig' in stats['perturbation']:
            stats['perturbation']['ig'] = {}
        stats['perturbation']['ig']['rationale'] = make_stats_dict(rationale_diffs_ig_vals)
        stats['perturbation']['ig']['nonrationale'] = make_stats_dict(nonrationale_diffs_ig_vals)
        stats['perturbation']['ig']['ranksum_statistic'] = float(rs_stat_pert_ig)
        stats['perturbation']['ig']['ranksum_pvalue'] = float(rs_p_pert_ig)

        if VERBOSE:
            print('Ranksums test comparing perturbation deltas from',
                    'Integrated Gradients rationale vs non-rationale',
                    'stat=%f, p=%f' % (rs_stat_pert_ig, rs_p_pert_ig))
        pert_hist_ig = [(rationale_diffs_ig_vals, 25, 'Rationale'),
                        (nonrationale_diffs_ig_vals, 25, 'Non-rationale')]
        visualizations.plot_hist(pert_hist_ig,
                                 title='Perturbation Deltas, Integrated Gradients',
                                 xlabel='original prediction - prediction without word',
                                 ylabel='Normalized Frequency',
                                 normed=True,
                                 savepath=os.path.join(OUT_DIR,
                                     'perturbation_hist_ig.png'))

        rationale_diffs_top_ig = []
        nonrationale_diffs_top_ig = []
        for seq in container.get_sequences():
            rationale_diffs, nonrationale_diffs = seq.perturbation_rationale(
                model,
                [seq.get_rationales(IG_TOP_RATIONALE_KEY)[0]])
            rationale_diffs_top_ig.append(rationale_diffs)
            nonrationale_diffs_top_ig.append(nonrationale_diffs)

        rationale_diffs_top_ig_vals = np.concatenate(rationale_diffs_top_ig).ravel()
        nonrationale_diffs_top_ig_vals = np.concatenate(nonrationale_diffs_top_ig).ravel()
        rs_stat_pert_top_ig, rs_p_pert_top_ig = ranksums(
            rationale_diffs_top_ig_vals,
            nonrationale_diffs_top_ig_vals)

        if not 'top_ig' in stats['perturbation']:
            stats['perturbation']['top_ig'] = {}
        stats['perturbation']['top_ig']['rationale'] = make_stats_dict(
            rationale_diffs_top_ig_vals)
        stats['perturbation']['top_ig']['nonrationale'] = make_stats_dict(
            nonrationale_diffs_top_ig_vals)
        stats['perturbation']['top_ig']['ranksum_statistic'] = float(
            rs_stat_pert_top_ig)
        stats['perturbation']['top_ig']['ranksum_pvalue'] = float(
            rs_p_pert_top_ig)

        if VERBOSE:
            print('Ranksums test comparing perturbation deltas from',
                    'Top IG rationale vs non-rationale',
                    'stat=%f, p=%f' % \
                    (rs_stat_pert_top_ig, rs_p_pert_top_ig))
        pert_hist_top_ig = [(rationale_diffs_top_ig_vals, 25, 'Rationale'),
                                (nonrationale_diffs_top_ig_vals, 25, 'Non-rationale')]
        visualizations.plot_hist(pert_hist_top_ig,
                                 title='Perturbation Deltas, Top IG',
                                 xlabel='original prediction - prediction without word',
                                 ylabel='Normalized Frequency',
                                 normed=True,
                                 savepath=os.path.join(OUT_DIR,
                                     'perturbation_hist_top_ig.png'))

        rationale_diffs_ig_fixed_length = []
        nonrationale_diffs_ig_fixed_length = []
        rationale_diffs_lime_fixed_length = []
        nonrationale_diffs_lime_fixed_length = []
        rationale_diffs_pert_fixed_length = []
        nonrationale_diffs_pert_fixed_length = []
        for seq in container.get_sequences():
            rationale_diffs, nonrationale_diffs = seq.perturbation_rationale(
                model,
                [seq.get_rationales(IG_FIXED_RATIONALE_KEY)[0]])
            rationale_diffs_ig_fixed_length.append(rationale_diffs)
            nonrationale_diffs_ig_fixed_length.append(nonrationale_diffs)

            rationale_diffs, nonrationale_diffs = seq.perturbation_rationale(
                model,
                [seq.get_rationales(LIME_FIXED_RATIONALE_KEY)[0]])
            rationale_diffs_lime_fixed_length.append(rationale_diffs)
            nonrationale_diffs_lime_fixed_length.append(nonrationale_diffs)

            rationale_diffs, nonrationale_diffs = seq.perturbation_rationale(
                model,
                [seq.get_rationales(PERTURB_FIXED_RATIONALE_KEY)[0]])
            rationale_diffs_pert_fixed_length.append(rationale_diffs)
            nonrationale_diffs_pert_fixed_length.append(nonrationale_diffs)

        rationale_diffs_ig_fixed_length_vals = np.concatenate(rationale_diffs_ig_fixed_length).ravel()
        nonrationale_diffs_ig_fixed_length_vals = np.concatenate(nonrationale_diffs_ig_fixed_length).ravel()
        rs_stat_pert_ig_fixed_length, rs_p_pert_ig_fixed_length = ranksums(
            rationale_diffs_ig_fixed_length_vals,
            nonrationale_diffs_ig_fixed_length_vals)
        if not 'ig_fixed_length' in stats['perturbation']:
            stats['perturbation']['ig_fixed_length'] = {}
        stats['perturbation']['ig_fixed_length']['rationale'] = make_stats_dict(
            rationale_diffs_ig_fixed_length_vals)
        stats['perturbation']['ig_fixed_length']['nonrationale'] = make_stats_dict(
            nonrationale_diffs_ig_fixed_length_vals)
        stats['perturbation']['ig_fixed_length']['ranksum_statistic'] = float(
            rs_stat_pert_ig_fixed_length)
        stats['perturbation']['ig_fixed_length']['ranksum_pvalue'] = float(
            rs_p_pert_ig_fixed_length)

        if VERBOSE:
            print('Ranksums test comparing perturbation deltas from',
                    'IG fixed length rationale vs non-rationale',
                    'stat=%f, p=%f' % \
                    (rs_stat_pert_ig_fixed_length, rs_p_pert_ig_fixed_length))
        pert_hist_ig_fixed_length = [(rationale_diffs_ig_fixed_length_vals, 25, 'Rationale'),
                                (nonrationale_diffs_ig_fixed_length_vals, 25, 'Non-rationale')]
        visualizations.plot_hist(pert_hist_ig_fixed_length,
                                 title='Perturbation Deltas, IG Fixed Length',
                                 xlabel='original prediction - prediction without word',
                                 ylabel='Normalized Frequency',
                                 normed=True,
                                 savepath=os.path.join(OUT_DIR,
                                     'perturbation_hist_ig_fixed_length.png'))

        rationale_diffs_lime_fixed_length_vals = np.concatenate(rationale_diffs_lime_fixed_length).ravel()
        nonrationale_diffs_lime_fixed_length_vals = np.concatenate(nonrationale_diffs_lime_fixed_length).ravel()
        rs_stat_pert_lime_fixed_length, rs_p_pert_lime_fixed_length = ranksums(
            rationale_diffs_lime_fixed_length_vals,
            nonrationale_diffs_lime_fixed_length_vals)
        if not 'lime_fixed_length' in stats['perturbation']:
            stats['perturbation']['lime_fixed_length'] = {}
        stats['perturbation']['lime_fixed_length']['rationale'] = make_stats_dict(
            rationale_diffs_lime_fixed_length_vals)
        stats['perturbation']['lime_fixed_length']['nonrationale'] = make_stats_dict(
            nonrationale_diffs_lime_fixed_length_vals)
        stats['perturbation']['lime_fixed_length']['ranksum_statistic'] = float(
            rs_stat_pert_lime_fixed_length)
        stats['perturbation']['lime_fixed_length']['ranksum_pvalue'] = float(
            rs_p_pert_lime_fixed_length)

        if VERBOSE:
            print('Ranksums test comparing perturbation deltas from',
                    'LIME fixed length rationale vs non-rationale',
                    'stat=%f, p=%f' % \
                    (rs_stat_pert_lime_fixed_length, rs_p_pert_lime_fixed_length))
        pert_hist_lime_fixed_length = [(rationale_diffs_lime_fixed_length_vals, 25, 'Rationale'),
                                (nonrationale_diffs_lime_fixed_length_vals, 25, 'Non-rationale')]
        visualizations.plot_hist(pert_hist_lime_fixed_length,
                                 title='Perturbation Deltas, LIME Fixed Length',
                                 xlabel='original prediction - prediction without word',
                                 ylabel='Normalized Frequency',
                                 normed=True,
                                 savepath=os.path.join(OUT_DIR,
                                     'perturbation_hist_lime_fixed_length.png'))

        rationale_diffs_pert_fixed_length_vals = np.concatenate(rationale_diffs_pert_fixed_length).ravel()
        nonrationale_diffs_pert_fixed_length_vals = np.concatenate(nonrationale_diffs_pert_fixed_length).ravel()
        rs_stat_pert_pert_fixed_length, rs_p_pert_pert_fixed_length = ranksums(
            rationale_diffs_pert_fixed_length_vals,
            nonrationale_diffs_pert_fixed_length_vals)
        if not 'pert_fixed_length' in stats['perturbation']:
            stats['perturbation']['pert_fixed_length'] = {}
        stats['perturbation']['pert_fixed_length']['rationale'] = make_stats_dict(
            rationale_diffs_pert_fixed_length_vals)
        stats['perturbation']['pert_fixed_length']['nonrationale'] = make_stats_dict(
            nonrationale_diffs_pert_fixed_length_vals)
        stats['perturbation']['pert_fixed_length']['ranksum_statistic'] = float(
            rs_stat_pert_pert_fixed_length)
        stats['perturbation']['pert_fixed_length']['ranksum_pvalue'] = float(
            rs_p_pert_pert_fixed_length)

        if VERBOSE:
            print('Ranksums test comparing perturbation deltas from',
                    'Pert. fixed length rationale vs non-rationale',
                    'stat=%f, p=%f' % \
                    (rs_stat_pert_pert_fixed_length, rs_p_pert_pert_fixed_length))
        pert_hist_pert_fixed_length = [(rationale_diffs_pert_fixed_length_vals, 25, 'Rationale'),
                                (nonrationale_diffs_pert_fixed_length_vals, 25, 'Non-rationale')]
        visualizations.plot_hist(pert_hist_pert_fixed_length,
                                 title='Perturbation Deltas, Pert. Fixed Length',
                                 xlabel='original prediction - prediction without word',
                                 ylabel='Normalized Frequency',
                                 normed=True,
                                 savepath=os.path.join(OUT_DIR,
                                     'perturbation_hist_pert_fixed_length.png'))

        rs_stat_pert_bothrationales_sis_vs_ig, rs_p_pert_bothrationales_sis_vs_ig = ranksums(
            rationale_diffs_sis_vals,
            rationale_diffs_ig_vals)
        if VERBOSE:
            print('Ranksums test comparing perturbation deltas from',
                    'SIS rationales vs IG rationales', 'stat=%f, p=%f' % \
                    (rs_stat_pert_bothrationales_sis_vs_ig,
                     rs_p_pert_bothrationales_sis_vs_ig))
        rs_stat_pert_bothrationales_sis_vs_top_ig, rs_p_pert_bothrationales_sis_vs_top_ig = ranksums(
            rationale_diffs_sis_vals,
            rationale_diffs_top_ig_vals)
        if VERBOSE:
            print('Ranksums test comparing perturbation deltas from',
                    'SIS rationales vs Top IG rationales', 'stat=%f, p=%f' % \
                    (rs_stat_pert_bothrationales_sis_vs_top_ig,
                     rs_p_pert_bothrationales_sis_vs_top_ig))
        pert_hist_allrationales = [(rationale_diffs_sis_vals, 25, 'SIS Rationales'),
                                   (rationale_diffs_ig_vals, 25, 'Integrated Gradients Rationales'),
                                   (rationale_diffs_top_ig_vals, 25, 'Top IG Rationales')]
        visualizations.plot_hist(pert_hist_allrationales,
                                 title='Perturbation Deltas, Rationales Only',
                                 xlabel='original prediction - prediction without word',
                                 ylabel='Normalized Frequency',
                                 normed=True,
                                 savepath=os.path.join(OUT_DIR,
                                     'perturbation_hist_allrationales.png'))

        if not 'bothrationales' in stats['perturbation']:
            stats['perturbation']['bothrationales'] = {}
            stats['perturbation']['bothrationales']['sis_vs_ig'] = {}
            stats['perturbation']['bothrationales']['sis_vs_top_ig'] = {}
        stats['perturbation']['bothrationales']['sis_vs_ig']['ranksum_statistic'] = \
            float(rs_stat_pert_bothrationales_sis_vs_ig)
        stats['perturbation']['bothrationales']['sis_vs_ig']['ranksum_pvalue'] = \
            float(rs_p_pert_bothrationales_sis_vs_ig)
        stats['perturbation']['bothrationales']['sis_vs_top_ig']['ranksum_statistic'] = \
            float(rs_stat_pert_bothrationales_sis_vs_top_ig)
        stats['perturbation']['bothrationales']['sis_vs_top_ig']['ranksum_pvalue'] = \
            float(rs_p_pert_bothrationales_sis_vs_top_ig)

        # HTML output visualizing SIS and IG rationales
        html_path = os.path.join(OUT_DIR, 'rationales.html')

        header = '''
        <h2>TF Binding Rationales (test set)</h2>
        <p>Highlighting rationales identified by SIS and integrated gradients.
        Only showing examples in test set where prediction >= threshold (default >= 90th percentile).
        Highlight shade indicates base importance, darker means more important, with linear scaling.</p>
        '''
        header += '<p>Task: %s</p>' % TASK
        header += '<p>Dataset: %s</p>' % DATASET
        header += '<p>Threshold: %.5f</p>' % container.get_threshold()
        html = ''
        for seq in container.get_sequences():
            x = seq.get_x()
            decoded_seq = decode_seq(x)
            html += '<p>Example %d</p>' % seq.i
            html += '<p>Predicted: %.3f  /  Actual: %.3f<br>' % \
                        (seq.original_prediction, y_test[seq.i])
            sis_rationales = seq.get_rationales(SIS_RATIONALE_KEY)
            ig_rationale = seq.get_rationales(IG_SUFF_RATIONALE_KEY)[0]
            ig_top_rationale = seq.get_rationales(IG_TOP_RATIONALE_KEY)[0]
            # Rationale Lengths
            html += 'Rationale length SIS = %s<br>' % \
                        str([len(r) for r in sis_rationales])
            html += 'Rationale length IG = %d<br>' % len(ig_rationale)
            html += 'Rationale length Top IG = %d<br>' % len(ig_top_rationale)
            # SIS Rationales
            html += '<p>SIS:</p>'
            for i, sis_rationale in enumerate(sis_rationales):
                if len(sis_rationale) > 1:
                    html += '<p><i>Sufficient Rationale %d:</i></p>' % (i + 1)
                html += visualizations.highlight_annot_tf(decoded_seq,
                    sis_rationale)
            # IG Rationale
            html += '<p>(Suff.) Integrated Gradients:</p>'
            html += visualizations.highlight_annot_tf(decoded_seq,
                ig_rationale)
            # Top IG Rationale
            html += '<p>Top IG:</p>'
            html += visualizations.highlight_annot_tf(decoded_seq,
                ig_top_rationale)
            html += '<hr>'

        visualizations.save_html(html, html_path, header)

        if VERBOSE:
            print('Rationales html dumped to file.\n')

        # Dump rationales to text file, replace non-rationale chars with 'N'
        rationale_texts_sis = []
        rationale_texts_ig = []
        rationale_texts_top_ig = []
        rationale_texts_fixed_length_ig = []
        rationale_texts_fixed_length_lime = []
        rationale_texts_fixed_length_pert = []

        sis_rationales_path = os.path.join(OUT_DIR, 'rationales_sis.txt')
        ig_rationales_path = os.path.join(OUT_DIR, 'rationales_ig.txt')
        top_ig_rationales_path = os.path.join(OUT_DIR,
            'rationales_top_ig.txt')
        fixed_length_ig_rationales_path = os.path.join(OUT_DIR, 'rationales_fixed_length_ig.txt')
        fixed_length_lime_rationales_path = os.path.join(OUT_DIR, 'rationales_fixed_length_lime.txt')
        fixed_length_pert_rationales_path = os.path.join(OUT_DIR, 'rationales_fixed_length_pert.txt')

        for seq in container.get_sequences():
            x = seq.get_x()
            decoded_seq = decode_seq(x)
            # SIS Rationales
            sis_rationales = seq.get_rationales(SIS_RATIONALE_KEY)
            for rationale in sis_rationales:
                rationale_text_sis = plaintext_rationale(decoded_seq,
                                            rationale.get_elms(),
                                            nonrationale='N')
                rationale_texts_sis.append((seq.i, rationale_text_sis))
            # IG Rationale
            ig_rationale = seq.get_rationales(IG_SUFF_RATIONALE_KEY)[0]
            rationale_text_ig = plaintext_rationale(decoded_seq,
                                    ig_rationale.get_elms(),
                                    nonrationale='N')
            rationale_texts_ig.append((seq.i, rationale_text_ig))
            # Top IG Rationale
            top_ig_rationale = seq.get_rationales(
                IG_TOP_RATIONALE_KEY)[0]
            rationale_text_top_ig = plaintext_rationale(decoded_seq,
                                    top_ig_rationale.get_elms(),
                                    nonrationale='N')
            rationale_texts_top_ig.append((seq.i, rationale_text_top_ig))
            # Fixed length IG Rationale
            fixed_length_ig_rationale = seq.get_rationales(IG_FIXED_RATIONALE_KEY)[0]
            rationale_text_fixed_length_ig = plaintext_rationale(decoded_seq,
                                    fixed_length_ig_rationale.get_elms(),
                                    nonrationale='N')
            rationale_texts_fixed_length_ig.append((seq.i, rationale_text_fixed_length_ig))
            # Fixed length LIME Rationale
            fixed_length_lime_rationale = seq.get_rationales(LIME_FIXED_RATIONALE_KEY)[0]
            rationale_text_fixed_length_lime = plaintext_rationale(decoded_seq,
                                    fixed_length_lime_rationale.get_elms(),
                                    nonrationale='N')
            rationale_texts_fixed_length_lime.append((seq.i, rationale_text_fixed_length_lime))
            # Fixed length Pert Rationale
            fixed_length_pert_rationale = seq.get_rationales(PERTURB_FIXED_RATIONALE_KEY)[0]
            rationale_text_fixed_length_pert = plaintext_rationale(decoded_seq,
                                    fixed_length_pert_rationale.get_elms(),
                                    nonrationale='N')
            rationale_texts_fixed_length_pert.append((seq.i, rationale_text_fixed_length_pert))

        save_rationale_texts(rationale_texts_sis, sis_rationales_path)
        save_rationale_texts(rationale_texts_ig, ig_rationales_path)
        save_rationale_texts(rationale_texts_top_ig,
            top_ig_rationales_path)
        save_rationale_texts(rationale_texts_fixed_length_ig,
            fixed_length_ig_rationales_path)
        save_rationale_texts(rationale_texts_fixed_length_lime,
            fixed_length_lime_rationales_path)
        save_rationale_texts(rationale_texts_fixed_length_pert,
            fixed_length_pert_rationales_path)

        if VERBOSE:
            print('Rationale texts dumped to file.\n')

        # Aligning rationales to known motifs (from JASPAR), compute log
        #   likelihood per base under the model of the known motif
        motifs_map = load_motifs_map(KNOWN_MOTIFS_MAP_PATH)
        if DATASET in motifs_map:
            if VERBOSE:
                print('Have known motif. Aligning and finding likelihood.')

            meme_path = os.path.join(KNOWN_MOTIFS_MEME_PATH, motifs_map[DATASET])
            motif = parse_meme(meme_path, replace_zeros_eps=1e-6)

            # Add motif entropy into stats dict
            stats['known_motif'] = {}
            stats['known_motif']['entropy_sum'] = motif_entropy(motif)

            replacement = make_replacement_embedding()

            align_scores_sis = []
            for i, rationale in rationale_texts_sis:
                rationale_encoded = encode_rationale(rationale)
                alignment_res = align_motif_rationale(motif, rationale_encoded,
                                                        normalize=True)
                align_start_idx = alignment_res[0]
                padded_motif = pad_motif(motif, align_start_idx, replacement)
                kl_vals = kl_div_motif_rationale(padded_motif, rationale_encoded)
                align_score = np.sum(kl_vals)
                align_scores_sis.append(align_score)

            align_scores_ig = []
            for i, rationale in rationale_texts_ig:
                rationale_encoded = encode_rationale(rationale)
                alignment_res = align_motif_rationale(motif, rationale_encoded,
                                                        normalize=True)
                align_start_idx = alignment_res[0]
                padded_motif = pad_motif(motif, align_start_idx, replacement)
                kl_vals = kl_div_motif_rationale(padded_motif, rationale_encoded)
                align_score = np.sum(kl_vals)
                align_scores_ig.append(align_score)

            align_scores_top_ig = []
            for i, rationale in rationale_texts_top_ig:
                rationale_encoded = encode_rationale(rationale)
                alignment_res = align_motif_rationale(motif, rationale_encoded,
                                                        normalize=True)
                align_start_idx = alignment_res[0]
                padded_motif = pad_motif(motif, align_start_idx, replacement)
                kl_vals = kl_div_motif_rationale(padded_motif, rationale_encoded)
                align_score = np.sum(kl_vals)
                align_scores_top_ig.append(align_score)

            align_scores_fixed_length_ig = []
            for i, rationale in rationale_texts_fixed_length_ig:
                rationale_encoded = encode_rationale(rationale)
                alignment_res = align_motif_rationale(motif, rationale_encoded,
                                                        normalize=True)
                align_start_idx = alignment_res[0]
                padded_motif = pad_motif(motif, align_start_idx, replacement)
                kl_vals = kl_div_motif_rationale(padded_motif, rationale_encoded)
                align_score = np.sum(kl_vals)
                align_scores_fixed_length_ig.append(align_score)

            align_scores_fixed_length_lime = []
            for i, rationale in rationale_texts_fixed_length_lime:
                rationale_encoded = encode_rationale(rationale)
                alignment_res = align_motif_rationale(motif, rationale_encoded,
                                                        normalize=True)
                align_start_idx = alignment_res[0]
                padded_motif = pad_motif(motif, align_start_idx, replacement)
                kl_vals = kl_div_motif_rationale(padded_motif, rationale_encoded)
                align_score = np.sum(kl_vals)
                align_scores_fixed_length_lime.append(align_score)

            align_scores_fixed_length_pert = []
            for i, rationale in rationale_texts_fixed_length_pert:
                rationale_encoded = encode_rationale(rationale)
                alignment_res = align_motif_rationale(motif, rationale_encoded,
                                                        normalize=True)
                align_start_idx = alignment_res[0]
                padded_motif = pad_motif(motif, align_start_idx, replacement)
                kl_vals = kl_div_motif_rationale(padded_motif, rationale_encoded)
                align_score = np.sum(kl_vals)
                align_scores_fixed_length_pert.append(align_score)

            alignment_hist_data = [(np.array(align_scores_sis),
                                        25, 'SIS'),
                                   (np.array(align_scores_ig),
                                        25, 'IG'),
                                   (np.array(align_scores_top_ig),
                                        25, 'Top IG')]
            visualizations.plot_hist(alignment_hist_data,
                title='Scoring Rationales against Known Motifs',
                xlabel='sum(KL divergence) over positions of after optimal known motif alignment',
                ylabel='Normalized Frequency',
                normed=True,
                legend_loc='upper right',
                savepath=os.path.join(OUT_DIR, 'motif_alignment_hist.png'))

            # Add alignment scores summary statistics to stats dict
            align_scores_sis_stats = make_stats_dict(align_scores_sis)
            align_scores_ig_stats = make_stats_dict(align_scores_ig)
            align_scores_top_ig_stats = make_stats_dict(align_scores_top_ig)
            align_scores_fixed_length_ig_stats = make_stats_dict(align_scores_fixed_length_ig)
            align_scores_fixed_length_lime_stats = make_stats_dict(align_scores_fixed_length_lime)
            align_scores_fixed_length_pert_stats = make_stats_dict(align_scores_fixed_length_pert)

            rs_stat_alignment_sis_vs_ig, rs_p_alignment_sis_vs_ig = ranksums(
                align_scores_sis,
                align_scores_ig)
            rs_stat_alignment_sis_vs_top_ig, rs_p_alignment_sis_vs_top_ig = ranksums(
                align_scores_sis,
                align_scores_top_ig)

            if 'alignment' not in stats:
                stats['alignment'] = {}
            stats['alignment']['sis'] = align_scores_sis_stats
            stats['alignment']['ig'] = align_scores_ig_stats
            stats['alignment']['fixed_length_ig'] = align_scores_fixed_length_ig_stats
            stats['alignment']['fixed_length_lime'] = align_scores_fixed_length_lime_stats
            stats['alignment']['fixed_length_pert'] = align_scores_fixed_length_pert_stats
            stats['alignment']['top_ig'] = align_scores_top_ig_stats
            stats['alignment']['sis_vs_ig'] = {}
            stats['alignment']['sis_vs_ig']['ranksum_statistic'] = float(rs_stat_alignment_sis_vs_ig)
            stats['alignment']['sis_vs_ig']['ranksum_pvalue'] = float(rs_p_alignment_sis_vs_ig)
            stats['alignment']['sis_vs_top_ig'] = {}
            stats['alignment']['sis_vs_top_ig']['ranksum_statistic'] = float(rs_stat_alignment_sis_vs_top_ig)
            stats['alignment']['sis_vs_top_ig']['ranksum_pvalue'] = float(rs_p_alignment_sis_vs_top_ig)

            if VERBOSE:
                print('Done with motif/rationale alignment.\n')

        # Add idxs for "interesting" test examples to stats dict
        stats['idxs'] = container.get_idxs()

        # Save stats dictionary to json
        save_stats(stats, stats_path)
        if VERBOSE:
            print('Dumped stats to json file.\n')

if __name__ == '__main__':
    main()


from django.shortcuts import render, redirect
from .forms import ServeursForms
from . import models
from django.http import HttpResponseRedirect
from fpdf import FPDF
from django.http import FileResponse

def ajout(request):
    form = ServeursForms
    return render(request, "adminsite/serveurs/ajout.html", {"form": form})
def traitement(request):
    form = ServeursForms(request.POST)
    if form.is_valid():
        donnees = form.save(commit=False)
        donnees.stockage_initial = donnees.stockage
        donnees.save()
        return HttpResponseRedirect("/adminsite/serveurs/")
    else:
        return render(request, "adminsite/serveurs/ajout.html", {"form": form})

def index(request):
    app = list(models.Serveurs.objects.all())
    return render(request, 'adminsite/serveurs/affiche.html',{"liste": app})

def delete(request, id):
    stream = models.Serveurs.objects.get(pk=id)
    stream.delete()
    return HttpResponseRedirect("/adminsite/serveurs/")

def update_traitement(request, id):
    serv = models.Serveurs.objects.get(pk=id)

    serv.stockage_initial = str(serv.stockage_initial)
    if serv.stockage_initial.endswith("Go"):
        serv.stockage_initial = float(serv.stockage_initial[:-2]) * 1_000

    elif serv.stockage_initial.endswith("To"):
        serv.stockage_initial = float(serv.stockage_initial[:-2]) * 1_000_000
    serv.stockage_initial= float(serv.stockage_initial)

    serv.stockage = str(serv.stockage)
    if serv.stockage.endswith("Go"):
        serv.stockage = float(serv.stockage[:-2]) * 1_000

    elif serv.stockage.endswith("To"):
        serv.stockage = float(serv.stockage[:-2]) * 1_000_000
    serv.stockage= float(serv.stockage)

    stockage_service = serv.stockage_initial - serv.stockage

    mform = ServeursForms(request.POST)
    if mform.is_valid():
        serveur = mform.save(commit=False)

        serveur.stockage = str(serveur.stockage)
        if serveur.stockage.endswith("Go"):
            serveur.stockage = float(serveur.stockage[:-2]) * 1_000

        elif serveur.stockage.endswith("To"):
            serveur.stockage = float(serveur.stockage[:-2]) * 1_000_000
        serveur.stockage = float(serveur.stockage)

        if serveur.stockage < stockage_service:
            return render(request, "adminsite/serveurs/update.html", {"form": mform, "error": "Erreur pas assez de stockage sur le serveur","id": id})
        else:
            serveur.stockage_initial = serveur.stockage
            serveur.stockage = serveur.stockage - stockage_service
            if serveur.stockage < 1_000_000:
                serveur.stockage = serveur.stockage / 1000
                serveur.stockage = int(serveur.stockage)
                serveur.stockage = str(serveur.stockage) + "Go"
            else:
                serveur.stockage = serveur.stockage / 1_000_000
                serveur.stockage = int(serveur.stockage)
                serveur.stockage = str(serveur.stockage) + "To"

            if serveur.stockage_initial < 1_000_000:
                serveur.stockage_initial = serveur.stockage_initial / 1000
                serveur.stockage_initial = int(serveur.stockage_initial)
                serveur.stockage_initial = str(serveur.stockage_initial) + "Go"
            else:
                serveur.stockage_initial = serveur.stockage_initial / 1_000_000
                serveur.stockage_initial = int(serveur.stockage_initial)
                serveur.stockage_initial = str(serveur.stockage_initial) + "To"
            serveur.id = id
            serveur.save()
        return HttpResponseRedirect("/adminsite/serveurs/")
    else:
        return render(request, "adminsite/serveurs/update_affiche.html", {"form": mform, "id": id})

def update(request, id):
    serv = models.Serveurs.objects.get(pk=id)
    x = serv.stockage_initial
    serv.stockage_initial = serv.stockage
    serv.stockage = x
    form = ServeursForms(serv.dico())
    return render(request,"adminsite/serveurs/update.html",{"form":form, "id": id})

def affiche(request, id):
    var = models.Serveurs.objects.get(pk=id)
    return render(request,"adminsite/serveurs/affiche.html",{"util": var, "id": id})

def serveur_pdf(request,id):
    serveur = models.Serveurs.objects.get(pk=id)
    pdf = FPDF()
    pdf.add_page()
    pdf.set_font("Arial", size=20)
    pdf.set_text_color(0, 0, 255)
    pdf.cell(200, 10,  ln=1, align='C')
    pdf.cell(200, 10, txt="RAPPORT SUR LE SERVEUR : " + serveur.nom, ln=1, align='C')
    pdf.cell(200, 10,  ln=1, align='C')
    pdf.set_font("Arial", size=15)
    pdf.cell(200, 10, txt="* Le serveur " + serveur.nom + " est un serveur de type " + models.Types.objects.get(
        pk=serveur.type.id).type, ln=2, align='L')
    pdf.cell(200, 10, txt="* Espace stockage: " + str(serveur.stockage_initial), ln=4, align='L')
    pdf.cell(200, 10, txt="* Espace restant: " + str(serveur.stockage), ln=5, align='L')
    pdf.cell(200, 10, txt="* Memoire vive: " + str(serveur.memoire), ln=6, align='L')
    pdf.cell(200, 10, txt="* Processeur: " + str(serveur.processeur), ln=8, align='L')
    line = 9
    pdf.cell(200, 10,  ln=1, align='C')
    if len(list(models.Applications.objects.filter(serveur=serveur))) == 0:
        pdf.cell(200, 10, txt="Il n'y a aucune application sur ce serveur.", ln=line, align='L')
        line = line + 1
    else:
        pdf.cell(200, 10, txt="* Les applications sur ce serveur:", ln=line, align='L')
        line = line + 1
        for i in list(models.Applications.objects.filter(serveur=serveur)):
            pdf.cell(200, 10, txt='         - ' +i.nom, ln=line, align='L')
            line = line + 1
    pdf.cell(200, 10,  ln=1, align='C')
    if len(list(models.Services.objects.filter(serveur_lancement=serveur))) == 0:
        pdf.cell(200, 10, txt="Il n'y a aucun service sur ce serveur.", ln=line, align='L')
        line = line + 1
    else:
        pdf.cell(200, 10, txt="* Les services sur ce serveur:", ln=line, align='L')
        line = line + 1
        for i in list(models.Services.objects.filter(serveur_lancement=serveur)):
            pdf.cell(200, 10, txt='         -'+i.nom, ln=line, align='L')
            line = line + 1
    pdf.output("Serveur-" + str(id) + ".pdf")
    response = FileResponse(open("Serveur-" + str(id) + ".pdf", 'rb'))
    return response


import re

import cv2
import configparser
import json
import numpy as np
import threading
import gi

gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, Gdk, GdkPixbuf, Gio

class SettingsWindow(Gtk.Application):
    def __init__(self):
        super(SettingsWindow, self).__init__(application_id=None, flags=Gio.ApplicationFlags.FLAGS_NONE)
        self.coords_view_box = [[0.0, 0.0], [0.0, 0.0]]
        self.coords_analyse_box = [[0.0, 0.0], [0.0, 0.0]]
        self.img = GdkPixbuf.Pixbuf.new_from_file("example_img.png")
        self.load_area_data()

    def do_activate(self):
        builder = Gtk.Builder()
        builder.add_from_file("UI/settings_window.glade")
        builder.connect_signals(self)

        self.settings_window = builder.get_object("settings_window")
        self.radiobutton_camera = builder.get_object("radiobutton_from_camera")
        self.radiobutton_file = builder.get_object("radiobutton_from_file")
        self.spin_button = builder.get_object("camera_index_spin_button")
        self.file_chooser_button = builder.get_object("file_chooser_button")
        self.list_of_available_cameras = builder.get_object("list_available")
        self.list_of_working_cameras = builder.get_object("list_working")
        self.drawing_area = builder.get_object("mask_editor")
        self.toggle_view_box = builder.get_object("toggle_btn_choose_view_box")
        self.toggle_box_to_analyse = builder.get_object("toggle_btn_choose_box_to_analyse")

        self.area_accept_button = builder.get_object("area_accept_button")

        self.area_accept_button.connect("released", self.on_area_accept_button)
        self.toggle_view_box.connect("toggled", self.on_toggled, 1)
        self.toggle_box_to_analyse.connect("toggled", self.on_toggled, 2)
        self.toggle_box_to_analyse.set_active(True)

        self.drawing_area.add_events(Gdk.EventMask.BUTTON_PRESS_MASK)
        self.drawing_area.add_events(Gdk.EventMask.BUTTON1_MOTION_MASK)
        self.drawing_area.set_size_request(720, 576)
        self.drawing_area.queue_draw()
        self.load_settings()
        self.settings_window.show_all()

    def load_area_data(self):
        config = configparser.ConfigParser()
        config.read("settings.ini")
        self.coords_view_box = [[int(k) for k in i.split(", ")]
                                for i in config.get("area_coordinates", "coords_of_view_box")[2:-2].split("], [")]
        self.coords_analyse_box = [[int(k) for k in i.split(", ")]
                                   for i in
                                   config.get("area_coordinates", "coords_of_box_to_analyse")[2:-2].split("], [")]

    def on_area_accept_button(self, *_):
        config = configparser.ConfigParser()
        config.read("settings.ini")
        config.set("area_coordinates", "coords_of_view_box", str(self.coords_view_box))
        config.set("area_coordinates", "coords_of_box_to_analyse", str(self.coords_analyse_box))

        with open("settings.ini", 'w') as configfile:
            config.write(configfile)

    def on_toggled(self, obj, type_number):
        if type_number == 1 and self.toggle_view_box.get_active():
            self.toggle_box_to_analyse.set_active(False)
        elif type_number == 2 and self.toggle_box_to_analyse.get_active():
            self.toggle_view_box.set_active(False)

    def on_refresh_list_of_available_cameras(self, *_):
        is_working = True
        dev_port = 0
        working_ports = []
        available_ports = []
        while is_working and dev_port < 101:
            camera = cv2.VideoCapture(dev_port)
            if not camera.isOpened():
                is_working = False
            else:
                is_reading, img = camera.read()
                if is_reading:
                    working_ports.append(dev_port)
                else:
                    available_ports.append(dev_port)
            dev_port += 1

        if len(available_ports) == 0:
            self.list_of_available_cameras.set_text("None")
        else:
            tmp_str = ""
            for i in available_ports:
                tmp_str += str(i) + ' '
            self.list_of_available_cameras.set_text(tmp_str)

        if len(working_ports) == 0:
            self.list_of_working_cameras.set_text("None")
        else:
            tmp_str = ""
            for i in working_ports:
                tmp_str += str(i) + ' '
            self.list_of_working_cameras.set_text(tmp_str)

    def on_accept_changes(self, *_):
        config = configparser.ConfigParser()
        config.read("settings.ini")
        config.set("video_stream", "from_camera", str(self.radiobutton_camera.get_active()))
        config.set("video_stream", "from_file", str(self.radiobutton_file.get_active()))
        config.set("video_file", "video_file_name", str(self.file_chooser_button.get_filename()))
        config.set("video_camera", "video_camera_index", str(self.spin_button.get_value())[:-2])

        with open("settings.ini", 'w') as configfile:
            config.write(configfile)
        self.settings_window.destroy()

    def on_decline_changes(self, *_):
        self.settings_window.destroy()

    def load_settings(self):
        config = configparser.ConfigParser()
        config.read("settings.ini")

        if config["video_stream"]["from_camera"] == "True":
            self.radiobutton_camera.set_active(self)
        else:
            self.radiobutton_file.set_active(self)

        self.spin_button.set_value(int(config["video_camera"]["video_camera_index"]))
        self.file_chooser_button.set_filename(config["video_file"]["video_file_name"])

    def on_area_draw(self, widget, cr, *_):
        Gdk.cairo_set_source_pixbuf(cr, self.img, 0, 0)
        cr.paint()

        size_x = min(
            max(-self.coords_view_box[0][0], self.coords_view_box[1][0] - self.coords_view_box[0][0]),
            720 - self.coords_view_box[0][0]
        )
        size_y = min(
            max(-self.coords_view_box[0][1], self.coords_view_box[1][1] - self.coords_view_box[0][1]),
            576 - self.coords_view_box[0][1]
        )
        cr.set_source_rgba(0.0, 0.9, 0.0, 0.8)
        cr.rectangle(self.coords_view_box[0][0], self.coords_view_box[0][1], size_x, size_y)
        cr.stroke()

        size_x = min(
            max(-self.coords_analyse_box[0][0], self.coords_analyse_box[1][0] - self.coords_analyse_box[0][0]),
            720 - self.coords_analyse_box[0][0]
        )
        size_y = min(
            max(-self.coords_analyse_box[0][1], self.coords_analyse_box[1][1] - self.coords_analyse_box[0][1]),
            576 - self.coords_analyse_box[0][1]
        )
        cr.set_source_rgba(0.9, 0.9, 0.0, 0.8)
        cr.rectangle(self.coords_analyse_box[0][0], self.coords_analyse_box[0][1], size_x, size_y)
        cr.stroke()

    def on_area_pressed(self, widget, event):
        if event.type == Gdk.EventType.BUTTON_PRESS and event.button == 1:  # 1 == left mouse button
            if self.toggle_view_box.get_active():
                self.coords_view_box[0] = [int(event.x), int(event.y)]
            else:
                self.coords_analyse_box[0] = [int(event.x), int(event.y)]

    def on_area_motion_notify_event(self, widget, event):
        pos_x = min(max(0, int(event.x)), 720)
        pos_y = min(max(0, int(event.y)), 576)
        if self.toggle_view_box.get_active():
            self.coords_view_box[1] = [pos_x, pos_y]
        else:
            self.coords_analyse_box[1] = [pos_x, pos_y]
        self.drawing_area.queue_draw()


import sys
import cv2
from scipy.misc import imread
import matplotlib
matplotlib.use("TkAgg")
from matplotlib import pyplot as plt

class Formatter(object):
    def __init__(self, im):
        self.im = im
    def __call__(self, x, y):
        z = self.im.get_array()[int(y), int(x)]
        return 'x={:.01f}, y={:.01f}, z={}'.format(x, y, z)
try:
    from tkinter import Tk
    from tkinter.filedialog import askopenfilename
    root = Tk()
    root.withdraw()
    fname = askopenfilename()
    root.destroy()
except:
    fname = sys.argv[1]

# img = imread(fname, 'F')
img = cv2.imread(fname, cv2.IMREAD_UNCHANGED)
print('img has shape and dtype:', img.shape, img.dtype)

fig, ax = plt.subplots()
im = ax.imshow(img, interpolation='none')
ax.format_coord = Formatter(im)
plt.show()


import PySimpleGUI as sg
import sys, yt_dlp, music_tag

errors = 0
currently_downloading = False

def what_is(url):
    # Work around to printing errors to the console
    global errors
    errors = ""

    class ignoreErrors:
        @staticmethod
        def error(msg):
            global errors
            errors += msg

        @staticmethod
        def warning(msg):
            pass

        @staticmethod
        def debug(msg):
            pass

    ydl_opts = {'skip_download': True, 'ignoreerrors': True, 'quiet': True, "logger": ignoreErrors, 'extract_flat': True, 'playlistend': 2}

    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
        ydl.extract_info(url, False)

    if errors:
        return errors

    if 'playlist' in url:
        return 'playlist'
    else:
        return 'vid'

def set_audio_metadata(data, t):
    if t == 'v':
        f = music_tag.load_file(data['requested_downloads'][0]['filepath'])

        fields = ['track', 'artist', 'album', 'release_year']

        for i in fields:
            if data.get(f'{i}'):
                if i == 'track':
                    f['tracktitle'] = data['track']
                    print(data['track'])
                elif i == 'release_year':
                    f['year'] = data[f'{i}']
                    print(data[f'{i}'])
                else:
                    f[f'{i}'] = data[f'{i}']
                    print(data[f'{i}'])

        f.save()

    else:
        songs = data['entries']

        for song in songs:
            f = music_tag.load_file(song['requested_downloads'][0]['filepath'])

            fields = ['track', 'artist', 'album', 'release_year']

            for i in fields:
                if song.get(f'{i}'):
                    if i == 'track':
                        f['tracktitle'] = song['track']
                        print(song['track'])
                    elif i == 'release_year':
                        f['year'] = song[f'{i}']
                        print(song[f'{i}'])
                    else:
                        f[f'{i}'] = song[f'{i}']
                        print(song[f'{i}'])

            f.save()

def download_video(url, t, path):
    if t == 'Audio':
        ydl_opts = {
            'ffmpeg_location': 'C:/ffmpeg/bin',
            'format': 'bestaudio/best',
            'audioformat': 'mp3',
            'outtmpl': f'{path}/%(title)s.%(ext)s',
            'postprocessors': [{
                'key': 'FFmpegExtractAudio',
                'preferredcodec': 'mp3',
            }]
        }
    else:
        ydl_opts = {
            'format': 'mp4/bestvideo/best',
            'outtmpl': f'{path}/%(title)s.%(ext)s',
        }

    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
        metadata = ydl.extract_info(url)

    if t == 'Audio':
        set_audio_metadata(metadata, 'v')

def download_playlist(url, t, path, s, e):
    if t == 'Audio':
        ydl_opts = {
            'ffmpeg_location': 'C:/ffmpeg/bin',
            'format': 'bestaudio/best',
            'outtmpl': f'{path}/%(title)s.%(ext)s',
            'audioformat': 'mp3',
            'playliststart': s,
            'playlistend': e,
            'postprocessors': [{
                'key': 'FFmpegExtractAudio',
                'preferredcodec': 'mp3',
            }]
        }
    else:
        ydl_opts = {
            'format': 'mp4/bestvideo/best',
            'outtmpl': f'{path}/%(title)s.%(ext)s',
            'playliststart': s,
            'playlistend': e,
        }

    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
        metadata = ydl.extract_info(url)

    print(metadata)

    if t == 'Audio':
        set_audio_metadata(metadata, 'p')

# Create the Window
vid_layout = [[sg.Text('YouTube URL:', font=('Helvetica', 20)), sg.Input(key='url', font=('Helvetica', 20))]]
playlist_layout = [[sg.Text('Youtube Playlist:', font=('Helvetica', 20)),
                    sg.Input(key='list_url', font=('Helvetica', 20))],
                   [sg.Text('Start Index:', font=('Helvetica', 20)), sg.Push(),
                    sg.Input('Blank for first', key='start_idx', font=('Helvetica', 20), s=15), sg.Push(),
                    sg.Text('End Index:', font=('Helvetica', 20)), sg.Push(),
                    sg.Input('Blank for last', key='end_idx', font=('Helvetica', 20), s=15)]]

layout = [[sg.Text('Youtube Downloader', font=('Helvetica', 40))],
          [sg.Column(vid_layout, key='vid'), sg.Column(playlist_layout, key='playlist', visible=False)],
          [sg.Push(), sg.Text('Select Folder: ', font=('Helvetica', 20)),
           sg.Input(key='fpath', font=('Helvetica', 20), s=41), sg.FolderBrowse(font=('Helvetica', 15))],
          [sg.Combo(['Video', 'Audio'], key='type', font=('Helvetica', 20)),
           sg.Button('Download', font=('Helvetica', 15)),
           sg.Button('Playlist Toggle', key='toggle', font=('Helvetica', 15)),
           sg.Cancel('Exit', font=('Helvetica', 15))]]

window = sg.Window('Youtube Downloader', layout)
cur_layout = 'vid'

# Start loop
while True:
    # Read window
    event, values = window.read(timeout=1)
    if event == sg.WIN_CLOSED or event == 'Exit':
        break

    if event == 'toggle':
        if cur_layout == 'playlist':
            window['playlist'].update(visible=False)
            window['vid'].update(visible=True)
            cur_layout = 'vid'
        else:
            window['playlist'].update(visible=True)
            window['vid'].update(visible=False)
            cur_layout = 'playlist'

    if event == 'Download':
        if values['type'] and values['fpath']:
            print(what_is(values['url']))
            if cur_layout == what_is(values['url']) and cur_layout == 'vid':
                download_video(values['url'], values['type'], values['fpath'])
                print('yes')
            elif cur_layout == what_is(values['list_url']) and cur_layout == 'playlist':
                print('yes')
                start = None
                end = None
                if not values['start_idx']:
                    start = 1
                if not values['end_idx']:
                    end = -1

                if start is None:
                    try: start = int(values['start_idx'])
                    finally: pass
                if end is None:
                    try: end = int(values['end_idx'])
                    finally: pass

                if start is not None and end is not None:
                    download_playlist(values['list_url'], values['type'], values['fpath'], start, end)

window.close()
sys.exit()


from pyts import datasets
import numpy as np
from sklearn.utils import Bunch

# there are two ways to handle the dataset,the first way is to fetch it from the internet
# the second way is to
def fetch_all_dataset():
    dataset_list = datasets.ucr_dataset_list()
    for dataset in dataset_list:
        datasets.fetch_ucr_dataset(dataset, return_X_y=True)

def _load_ucr_dataset(dataset, path):
    """Load a UCR data set from a local folder.

    Parameters
    ----------
    dataset : str
        Name of the dataset.

    path : str
        The path of the folder containing the cached data set.

    Returns
    -------
    data : Bunch
        Dictionary-like object, with attributes:

        data_train : array of floats
            The time series in the training set.
        data_test : array of floats
            The time series in the test set.
        target_train : array
            The classification labels in the training set.
        target_test : array
            The classification labels in the test set.
        DESCR : str
            The full description of the dataset.
        url : str
            The url of the dataset.

    Notes
    -----
    Padded values are represented as NaN's.

    """
    new_path = path + dataset + '/'
    try:
        with(open(new_path + dataset + '.txt', encoding='utf-8')) as f:
            description = f.read()
    except UnicodeDecodeError:
        with(open(new_path + dataset + '.txt', encoding='ISO-8859-1')) as f:
            description = f.read()
    try:
        data_train = np.genfromtxt(new_path + dataset + '_TRAIN.txt')
        data_test = np.genfromtxt(new_path + dataset + '_TEST.txt')

        X_train, y_train = data_train[:, 1:], data_train[:, 0]
        X_test, y_test = data_test[:, 1:], data_test[:, 0]

    except IndexError:
        train = loadarff(new_path + dataset + '_TRAIN.arff')
        test = loadarff(new_path + dataset + '_TEST.arff')

        data_train = np.asarray([train[0][name] for name in train[1].names()])
        X_train = data_train[:-1].T.astype('float64')
        y_train = data_train[-1]

        data_test = np.asarray([test[0][name] for name in test[1].names()])
        X_test = data_test[:-1].T.astype('float64')
        y_test = data_test[-1]

    try:
        y_train = y_train.astype('float64').astype('int64')
        y_test = y_test.astype('float64').astype('int64')
    except ValueError:
        y_train = y_train.astype(str)
        y_test = y_test.astype(str)

    bunch = Bunch(
        data_train=X_train, target_train=y_train,
        data_test=X_test, target_test=y_test,
        DESCR=description,
        url=("http://www.timeseriesclassification.com/"
             "description.php?Dataset={}".format(dataset))
    )

    return bunch


#!/usr/bin/env python3
# coding: utf-8

import os
from lxml import etree
from selenium import webdriver
from LTAF_nightly_2_release import Case
import time
import argparse
from datetime import datetime, timedelta
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class WarningCase(Case):
	"""docstring for ClassName"""
	def __init__(self, arg):
		self.me = arg.xpath('.//text()')
		self.boardname = arg.xpath('.//..//td[2]/text()')
		self.bits = arg.xpath('.//..//td[3]/text()')
		self.cpu = arg.xpath('.//..//td[5]/text()')
		self.mode = arg.xpath('.//..//td[6]/text()')
		self.tcp_64 = arg.xpath('.//..//td[8]/text()')
		self.tcp_1024 = arg.xpath('.//..//td[10]/text()')
		self.tcp_65536 = arg.xpath('.//..//td[12]/text()')
		self.udp_1400 = arg.xpath('.//..//td[14]/text()')

	def display_result(self):
		super().set_attr_tostring()
		list_result = [
			self.boardname,
			self.bits,
			self.cpu,
			self.mode
		]
		#print(type(self.udp_1400))
		dict_result = {
			self.tcp_64 : 'TCP_64',
			self.tcp_1024 : 'TCP_1024',
			self.tcp_65536 : 'TCP_65536',
			self.udp_1400 : 'UDP_1400'
		}

		if self.me in dict_result:
			list_result.append(dict_result[self.me])
			list_result.append(self.me)
		return list_result
		#print(list_result)

class Result(object):
	def __init__(self, source):
		self.source = source

	def get_result(self):
		html = etree.HTML(self.source)
		cases = html.xpath('//td[@style="color: rgb(255, 165, 0);"]')
		#cases = html.xpath('//td[@style="color: rgb(255, 165, 0);"]/..//td[2]/text()')
		#cases = html.xpath('//td[@class="baseline"]/text()')
		return cases

def screen_short(date, capture_type=None):
	#SCREEN_PATH = '/net/pek-vx-nwk1/buildarea1/hyan1/ScreenShort'
	SCREEN_PATH = '/folk/hyan1/Nightly/nightlyReport'
	#chrome_options = webdriver.ChromeOptions()
	#chrome_options.add_argument('--headless')
	#chrome_options.add_argument('--hide-scrollbars')
	#browser =webdriver.Chrome(options = chrome_options)
	#browser.get('http://pek-vx-nwk1/report/{}'.format(date))
	options = webdriver.FirefoxOptions()
	options.add_argument('-headless')
	browser = webdriver.Firefox(executable_path='/buildarea1/hyan1/flask/geckodriver', firefox_options=options)
	browser.get('http://pek-vx-nwk1/warning/{}'.format(date))
	#print(browser.page_source)
	wait = WebDriverWait(browser, 20)
	item = wait.until(EC.presence_of_element_located((By.ID, 'up')))
	#time.sleep(2)
	# generate pdf
	#google-chrome --headless --disable-gpu --print-to-pdf=sreenshort_2020-02-28.pdf http://pek-vx-nwk1/report/2020-02-28

	width = browser.execute_script("return document.documentElement.scrollWidth")
	height = browser.execute_script("return document.documentElement.scrollHeight")
	browser.set_window_size(width, height)
	time.sleep(2)
	#browser.save_screenshot('{}/sreenshort_{}_{}.png'.format(SCREEN_PATH, capture_type, date))
	browser.save_screenshot('{}/sreenshort_warning_cases.png'.format(SCREEN_PATH))
	browser.close()

def get_warning_case(date):
	#chrome_options = webdriver.ChromeOptions()
	#chrome_options.add_argument('--headless')
	#browser = webdriver.Chrome(options = chrome_options)

	options = webdriver.FirefoxOptions()
	options.add_argument('-headless')
	browser = webdriver.Firefox(executable_path='/buildarea1/hyan1/flask/geckodriver', firefox_options=options)
	browser.get('http://pek-vx-nwk1/report/{}'.format(date))
	wait = WebDriverWait(browser, 20)
	item = wait.until(EC.presence_of_all_elements_located((By.XPATH, '//td[@style="color: rgb(255, 165, 0);"]')))
	page_source = browser.page_source
	#time.sleep(5)
	#print(page_source)
	nightly_result = Result(page_source)
	#print(nightly_result.get_result())
	waring_result = nightly_result.get_result()
	#print(waring_result)
	list_result = []
	for result in waring_result:
		#print(result)
		case = WarningCase(result)
		#case.get_attr()
		warning_case = case.display_result()
		list_result.append(warning_case)
	#print(list_result)
	browser.close()
	return list_result

def time_delta(run_date):
    y = datetime.strptime(run_date, '%Y-%m-%d')
    z = datetime.now()
    diff = z - y
    return diff.days - 1

if __name__ == '__main__':
	parse = argparse.ArgumentParser()
	parse.add_argument('--run_date', help='run date', dest='run_date', required=True)
	args = parse.parse_args()
	date = args.run_date
	if time_delta(date) <= 0:
		screen_short(date)


from django.conf.urls import patterns,url
from qtl import views

urlpatterns = patterns('',
	url(r'^trait/$', views.SearchTraitView, name = 'search_individual_trait'),
	url(r'^check/$', views.MultipleTraitView, name = 'search_multiple_trait'),
	url(r'^graph/$', views.Graphview, name='display_graph'),
	url(r'^godag/$', views.Graphview, name='display_godag'),
	url(r'^output/$', views.OutputDataView, name='display_data'),
	url(r'^storecsv/$', views.StoreCsvView, name='store_csv'),
						)