from __future__ import division
from collections import Counter
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from sklearn.metrics import log_loss
import pandas as pd
import numpy as np
import xgboost as xgb
import math
import nltk
from nltk import ngrams
from sklearn.cross_validation import train_test_split
from string import punctuation
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.preprocessing import MinMaxScaler
from simhash import Simhash
import plotly
import plotly.plotly as py
import plotly.graph_objs as go
from matplotlib import pyplot
from sklearn.manifold import TSNE
import pickle
from tqdm import tqdm, tqdm_pandas
tqdm_pandas(tqdm())
import spacy
nlp = spacy.load('en')

question_types = ["what", "how", "why", "is", "which", "can", "i", "who", "do", "where", "if", "does", "are", "when", "should", "will", "did", "has", "would", "have", "was", "could"]

def submit(p_test):

    sub = pd.DataFrame()

    sub['test_id'] = df_test['test_id']
    sub['is_duplicate'] = p_test

    sub.to_csv('simple_xgb.csv', index=False)

def get_inverse_freq(inverse_freq, count, min_count=2):

    if count < min_count:
        return 0
    else:
        return inverse_freq

def get_tf(text):

    tf = {}

    for word in text:
        tf[word] = text.count(word)/len(text)

    return tf

def tuple_similarity(q1_words, q2_words):

    if len(q1_words) == 0 or len(q2_words) == 0:
        return 0

    common_words = len(set(q1_words).intersection(set(q2_words)))
    all_words = len(set(q1_words).union(set(q2_words)))

    return common_words/all_words

def get_ne_score(row):

    q1_words = str(row.question1).lower().split()
    q2_words = str(row.question2).lower().split()

    # all_words_score = np.sum([weights.get(w, 0) for w in q1_words]) + np.sum([weights.get(w, 0) for w in q2_words])

    q1 = nlp(unicode(str(row["question1"]), "utf-8"))
    q2 = nlp(unicode(str(row["question2"]), "utf-8"))

    q1_ne = q1.ents
    q2_ne = q2.ents

    q1_ne = set([str(i) for i in q1_ne])
    q2_ne = set([str(i) for i in q2_ne])

    if len(q1_ne) == 0:
        q1_ne_ratio = 0
    else:
        q1_ne_ratio = len(q1_ne)/len(q1_words)

    if len(q2_ne) == 0:
        q2_ne_ratio = 0
    else:
        q2_ne_ratio = len(q2_ne)/len(q2_words)

    common_ne = list(q1_ne.intersection(q2_ne))
    # common_ne_weights = np.sum([weights.get(w, 0) for w in common_ne])

    if len(q1_ne) + len(q2_ne) == 0:
        common_ne_score = 0
    else:
       common_ne_score = len(common_ne)/(len(q1_words) + len(q2_words) - len(common_ne))

    return pd.Series({
        "q1_ne_ratio": q1_ne_ratio,
        "q2_ne_ratio": q2_ne_ratio,
        "ne_diff": abs(q1_ne_ratio - q2_ne_ratio),
        "ne_score": common_ne_score
    })

def basic_nlp(row):

    # q1_tf = get_tf(q1_words)
    # q2_tf = get_tf(q2_words)

    q1_words = str(row.question1).lower().split()
    q2_words = str(row.question2).lower().split()

    #modify this!
    if len(q1_words) == 0 or len(q2_words) == 0:
        return 0

    common_words = list(set(q1_words).intersection(q2_words))

    common_words_score = np.sum([weights.get(w, 0) for w in common_words])
    all_words_score = np.sum([weights.get(w, 0) for w in q1_words]) + np.sum([weights.get(w, 0) for w in q2_words]) - common_words_score

    hamming_score = sum(1 for i in zip(q1_words, q2_words) if i[0]==i[1])/max(len(q1_words), len(q2_words))

    jacard_score =  len(common_words)/(len(q1_words) + len(q2_words) - len(common_words))
    cosine_score = len(common_words)/(pow(len(q1_words),0.5)*pow(len(q2_words),0.5))

    bigrams_q1 = set(ngrams(q1_words, 2))
    bigrams_q2 = set(ngrams(q2_words, 2))
    common_bigrams = len(bigrams_q1.intersection(bigrams_q2))
    if common_bigrams == 0:
        bigram_score = 0
    else:
        bigram_score = common_bigrams/(len(bigrams_q1.union(bigrams_q2)))

    trigrams_q1 = set(ngrams(q1_words, 3))
    trigrams_q2 = set(ngrams(q2_words, 3))
    common_trigrams = len(trigrams_q1.intersection(trigrams_q2))
    if common_trigrams == 0:
        trigram_score = 0
    else:
        trigram_score = common_trigrams/(len(trigrams_q1.union(trigrams_q2)))

    # sequence1 = get_word_bigrams(q1_words)
    # sequence2 = get_word_bigrams(q2_words)

    # try:
    #     simhash_diff = Simhash(sequence1).distance(Simhash(sequence2))/64
    # except:
    #     simhash_diff = 0.5

    q1 = nlp(unicode(str(row["question1"]), "utf-8"))
    q2 = nlp(unicode(str(row["question2"]), "utf-8"))

    q1_ne = q1.ents
    q2_ne = q2.ents

    q1_ne = set([str(i) for i in q1_ne])
    q2_ne = set([str(i) for i in q2_ne])

    if len(q1_ne) == 0:
        q1_ne_ratio = 0
    else:
        q1_ne_ratio = len(q1_ne)/len(q1_words)

    if len(q2_ne) == 0:
        q2_ne_ratio = 0
    else:
        q2_ne_ratio = len(q2_ne)/len(q2_words)

    common_ne = list(q1_ne.intersection(q2_ne))
    # common_ne_weights = np.sum([weights.get(w, 0) for w in common_ne])

    if len(q1_ne) + len(q2_ne) == 0:
        common_ne_score = 0
    else:
       common_ne_score = len(common_ne)/(len(q1_words) + len(q2_words) - len(common_ne))

    pos_hash = {}
    common_pos = []

    for word in q1:
        if word.tag_ not in pos_hash:
            pos_hash.update({word.tag_ : [word.text]})
        else:
            pos_hash[word.tag_].append(word.text)

    for word in q2:
        if word.tag_ not in pos_hash:
            continue
        if word.text in pos_hash[word.tag_]:
            common_pos.append(word.text)

    common_pos_score = np.sum([weights.get(w, 0) for w in common_pos])
    all_pos_score = np.sum([weights.get(w, 0) for w in q1_words]) + np.sum([weights.get(w, 0) for w in q2_words]) - common_pos_score

    q1_pronouns_count = 0
    q2_pronouns_count = 0

    for word in q1:
        if str(word.tag_) == "PRP":
            q1_pronouns_count += 1

    for word in q2:
        if str(word.tag_) == "PRP":
            q2_pronouns_count += 1

    pronouns_diff = abs(q1_pronouns_count - q2_pronouns_count)

    q1_nc = q1.noun_chunks
    q2_nc = q2.noun_chunks

    q1_nc = set([str(i) for i in q1_nc])
    q2_nc = set([str(i) for i in q2_nc])

    common_nc = len(q1_nc.intersection(q2_nc))

    if len(q1_nc) + len(q2_nc) == 0:
        common_nc_score = 0
    else:
       common_nc_score = common_nc/(len(q1_nc) + len(q2_nc) - common_nc)

    fw_q1 = q1_words[0]
    fw_q2 = q2_words[0]

    if fw_q1 == fw_q2 and fw_q1 in question_types:
        question_type_same = 1
    else:
        question_type_same = 0

    try:
        q1_quotes = len(re.findall(r'\"(.+?)\"', row["question1"]))
    except:
        q1_quotes = 0

    try:
        q2_quotes = len(re.findall(r'\"(.+?)\"', row["question2"]))
    except:
        q2_quotes = 0

    # if len(q1_ne) == 0:
    #     q1_ne_hash_freq = 1
    # else:
    #     hash_key1 = hash("-".join(set([str(i).lower() for i in q1_ne])))

    #     if hash_key1 not in hash_table_ne:
    #         q1_ne_hash_freq = 1
    #     else:
    #         q1_ne_hash_freq = hash_table_ne[hash_key1]

    # if len(q2_ne) == 0:
    #     q2_ne_hash_freq = 1
    # else:
    #     hash_key2 = hash("-".join(set([str(i).lower() for i in q2_ne])))

    #     if hash_key2 not in hash_table_ne:
    #         q2_ne_hash_freq = 1
    #     else:
    #         q2_ne_hash_freq = hash_table_ne[hash_key2]

    try:
        q1_sents = len(nltk.tokenize.sent_tokenize(row.question1))
    except:
        q1_sents = 1
    try:
        q2_sents = len(nltk.tokenize.sent_tokenize(row.question2))
    except:
        q2_sents = 1

    q1_exclaim = sum([1 for i in str(row.question1) if i == "!"])
    q2_exclaim = sum([1 for i in str(row.question2) if i == "!"])

    q1_question = sum([1 for i in str(row.question1) if i == "?"])
    q2_question = sum([1 for i in str(row.question2) if i == "?"])

    hash_key1 = hash(str(row["question1"]).lower())
    if hash_key1 in hash_table:
        q1_hash_freq = hash_table[hash_key1]
    else:
        q1_hash_freq = 1

    hash_key2 = hash(str(row["question2"]).lower())
    if hash_key2 in hash_table:
        q2_hash_freq = hash_table[hash_key2]
    else:
        q2_hash_freq = 1

    # if hash_key1 in pos_hash_table:
    #     q1_dup_ratio = pos_hash_table[hash_key1]/q1_hash_freq
    # else:

    #     q1_dup_ratio = 0

    # if hash_key2 in pos_hash_table:
    #     q2_dup_ratio = pos_hash_table[hash_key2]/q2_hash_freq
    # else:
    #     q2_dup_ratio = 0

    spacy_sim = q1.similarity(q2)

    return pd.Series({

        "weighted_word_match_ratio" : common_words_score/all_words_score,
        "weighted_word_match_diff": all_words_score - common_words_score,
        "weighted_word_match_sum": common_words_score,
        "jacard_score": jacard_score,
        "hamming_score": hamming_score,
        "cosine_score": cosine_score,
        "bigram_score": bigram_score,
        "trigram_score": trigram_score,
        "pos_score": common_pos_score/all_pos_score,
        # "simhash_diff": simhash_diff,
        "question_type_same": question_type_same,
        "q1_stops": len(set(q1_words).intersection(stops))/len(q1_words),
        "q2_stops": len(set(q2_words).intersection(stops))/len(q2_words),
        "q1_len": len(str(row.question1)),
        "q2_len": len(str(row.question2)),
        "len_diff": abs(len(str(row.question1)) - len(str(row.question2))),
        "len_avg": (len(str(row.question1)) + len(str(row.question2)))/2,
        "q1_sents": q1_sents,
        "q2_sents": q2_sents,
        "sents_diff": abs(q1_sents - q2_sents),
        "q1_words": len(q1_words),
        "q2_words": len(q2_words),
        "words_diff": abs(len(q1_words) - len(q2_words)),
        "words_avg": (len(q1_words) + len(q2_words))/2,
        "q1_caps_count": sum([1 for i in str(row.question1) if i.isupper()]),
        "q2_caps_count": sum([1 for i in str(row.question2) if i.isupper()]),
        "q1_exclaim": q1_exclaim,
        "q2_exclaim": q2_exclaim,
        "exclaim_diff": abs(q1_exclaim - q2_exclaim),
        "q1_question": q1_question,
        "q2_question": q2_question,
        "question_diff": abs(q1_question - q2_question),
        "ne_score": common_ne_score,
        "nc_score": common_nc_score,
        "q1_ne_ratio": q1_ne_ratio,
        "q2_ne_ratio": q2_ne_ratio,
        "ne_diff": abs(q1_ne_ratio - q2_ne_ratio),
        "q1_quotes": q1_quotes,
        "q2_quotes": q2_quotes,
        "quotes_diff": abs(q1_quotes - q2_quotes),
        # "q1_ne_hash_freq": q1_ne_hash_freq,
        # "q2_ne_hash_freq": q2_ne_hash_freq,
        # "chunk_hash_diff": abs(q1_ne_hash_freq - q2_ne_hash_freq),
        "q1_hash_freq": q1_hash_freq,
        "q2_hash_freq": q2_hash_freq,
        "q_freq_avg": (q1_hash_freq + q2_hash_freq)/2,
        "freq_diff": abs(q1_hash_freq - q2_hash_freq),
        "spacy_sim": spacy_sim,
        "q1_pronouns_count": q1_pronouns_count,
        "q2_pronouns_count": q2_pronouns_count,
        "pronouns_diff": pronouns_diff
        # "q1_dup_ratio": q1_dup_ratio,
        # "q2_dup_ratio": q2_dup_ratio,
        # "q1_q2_dup_ratio_avg": (q1_dup_ratio + q2_dup_ratio)/2
    })

def neighbor_intersection(row):

    q1_neighbors = graph[row["question1"]]
    q2_neighbors = graph[row["question2"]]

    common_neighbors = set(q1_neighbors).intersection(q2_neighbors)

    return len(common_neighbors)

def get_q1_second_degree_freq(row):

    q1_neighbors = graph[row["question1"]]

    q1_second_degree_neighbors = []
    for i in q1_neighbors:
        q1_second_degree_neighbors += graph[i]

    return len(set(q1_second_degree_neighbors))

def get_q2_second_degree_freq(row):

    q2_neighbors = graph[row["question2"]]

    q2_second_degree_neighbors = []
    for i in q2_neighbors:
        q2_second_degree_neighbors += graph[i]

    return len(set(q2_second_degree_neighbors))

def second_degree_intersection(row):

    q1_neighbors = graph[row["question1"]]
    q2_neighbors = graph[row["question2"]]

    q1_second_degree_neighbors = []
    for i in q1_neighbors:
        q1_second_degree_neighbors += graph[i]

    q2_second_degree_neighbors = []
    for i in q2_neighbors:
        q2_second_degree_neighbors += graph[i]

    common_second_degree_neighbors = set(q1_second_degree_neighbors).intersection(set(q2_second_degree_neighbors))

    return len(common_second_degree_neighbors)

# def pos_neighbor_intersection(row):

#     if row["question1"] in pos_graph and row["question2"] in pos_graph:
#         q1_neighbors = pos_graph[row["question1"]]
#         q2_neighbors = pos_graph[row["question2"]]

#         common_neighbors = set(q1_neighbors).intersection(q2_neighbors)

#         return len(common_neighbors)/(len(q1_neighbors) + len(q2_neighbors) - len(common_neighbors))

#     else:
#         return 0

def get_word_bigrams(words):

    ngrams = []

    for i in range(0, len(words)):
        if i > 0:
            ngrams.append("%s %s"%(words[i-1], words[i]))

    return ngrams

def generate_hash_freq(row):

    hash_key1 = hash(row["question1"].lower())
    hash_key2 = hash(row["question2"].lower())

    if hash_key1 not in hash_table:
        hash_table[hash_key1] = 1
    else:
        hash_table[hash_key1] += 1

    if hash_key2 not in hash_table:
        hash_table[hash_key2] = 1
    else:
        hash_table[hash_key2] += 1

def generate_duplicate_freq(row):

    hash_key1 = hash(row["question1"].lower())
    hash_key2 = hash(row["question2"].lower())

    if hash_key1 not in pos_hash_table and row["is_duplicate"] == 1:
        pos_hash_table[hash_key1] = 1
    elif hash_key1 not in pos_hash_table and row["is_duplicate"] == 0:
        pos_hash_table[hash_key1] = 0
    elif hash_key1 in pos_hash_table and row["is_duplicate"] == 1:
        pos_hash_table[hash_key1] += 1
    # elif hash_key1 in pos_hash_table and row["is_duplicate"] == 0:
    #     pass

    if hash_key2 not in pos_hash_table and row["is_duplicate"] == 1:
        pos_hash_table[hash_key2] = 1
    elif hash_key2 not in pos_hash_table and row["is_duplicate"] == 0:
        pos_hash_table[hash_key2] = 0
    elif hash_key2 in pos_hash_table and row["is_duplicate"] == 1:
        pos_hash_table[hash_key2] += 1
    # elif hash_key1 in pos_hash_table and row["is_duplicate"] == 0:
    #     pass

def generate_positive_graph(row):

    hash_key1 = row["question1"]
    hash_key2 = row["question2"]

    if row["is_duplicate"] == 1:
        if hash_key1 not in pos_graph:
            pos_graph[hash_key1] = [hash_key2]
        elif hash_key1 in pos_graph:
            pos_graph[hash_key1].append(hash_key2)

        if hash_key2 not in pos_graph:
            pos_graph[hash_key2] = [hash_key1]
        elif hash_key2 in pos_graph:
            pos_graph[hash_key2].append(hash_key1)

def generate_graph_table(row):

    hash_key1 = row["question1"]
    hash_key2 = row["question2"]

    if hash_key1 not in graph:
        graph[hash_key1] = [hash_key2]
    elif hash_key1 in graph:
        graph[hash_key1].append(hash_key2)

    if hash_key2 not in graph:
        graph[hash_key2] = [hash_key1]
    elif hash_key2 in graph:
        graph[hash_key2].append(hash_key1)

def augment_rows():

    new_graph = graph

    for q1 in graph:

        q2_list = graph[q1]

        for i in q2_list:
            for j in q2_list:
                if i != j:
                    if j not in graph[i]:
                        new_graph[i].append(j)

    # new_df_train = df_train[["question1", "question2", "is_duplicate"]]

    # for i in new_graph:

def generate_ne_freq(row):

    q1 = nlp(unicode(str(row["question1"]), "utf-8"))
    q2 = nlp(unicode(str(row["question2"]), "utf-8"))

    q1_ne = q1.ents
    q2_ne = q2.ents

    q1_ne = "-".join(set([str(i).lower() for i in q1_ne]))
    q2_ne = "-".join(set([str(i).lower() for i in q2_ne]))

    hash_key1 = hash(q1_ne)
    hash_key2 = hash(q2_ne)

    if hash_key1 not in hash_table_ne:
        hash_table_ne[hash_key1] = 1
    else:
        hash_table_ne[hash_key1] += 1

    if hash_key2 not in hash_table_ne:
        hash_table_ne[hash_key2] = 1
    else:
        hash_table_ne[hash_key2] += 1

def oversample(x_train):

    neg_train = x_train[x_train.is_duplicate == 0]
    pos_train = x_train[x_train.is_duplicate == 1]

    #Oversampling negative class
    p = 0.165
    scale = ((len(pos_train) / (len(pos_train) + len(neg_train))) / p) - 1 #How much times greater is the train ratio than actual

    while scale > 1:
        neg_train = pd.concat([neg_train, neg_train])
        scale -=1
    neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train))]])

    return pd.concat([pos_train, neg_train])

#When plotted a histogram of degrees, only -1,1 and 2 are observed. Which means either you're max 2 degree separated or you're separate(with 5 as a cutoff).
#Add (number of second degree connections) and its intersection as a feature
def bfs(q_node, q_search, separation):

    if separation > 5:
        return -1

    if len(graph[q_node]) > 0:

        shortest_res = 32768

        if q_search in graph[q_node]:
            return separation
        else:

            for i,j in enumerate(graph[q_node]):

                if i > 5:
                    return shortest_res

                bfs_res = bfs(j, q_search, separation + 1)

                if bfs_res != -1 and bfs_res < shortest_res:
                    shortest_res = bfs_res

            return shortest_res

    else:
        return -1

def initialize_bfs(row):

    q1 = row["question1"]
    q2 = row["question2"]

    shortest_res = 32768

    for i in graph[q1]:
        if i != q2:
            res = bfs(i, q2, 1)

            if res != -1 and res < shortest_res:
                shortest_res = res

    if shortest_res == 32768:
        return -1
    else:
        return shortest_res

def augment_test(row):
    global new_df_test

    #map q1 with dups of q2
    if row["question2"] in pos_graph:
        new_rows = pd.DataFrame()
        q2_dups = pos_graph[row["question2"]]
        new_rows["question2"] = [i for i in q2_dups]
        new_rows["question1"] = row["question1"]
        new_rows["test_id"] = row["test_id"]
        new_df_test = pd.concat([new_df_test, new_rows])

    #map q2 with dups of q1
    if row["question1"] in pos_graph:
        new_rows = pd.DataFrame()
        q1_dups = pos_graph[row["question1"]]
        new_rows["question1"] = [i for i in q1_dups]
        new_rows["question2"] = row["question2"]
        new_rows["test_id"] = row["test_id"]
        new_df_test = pd.concat([new_df_test, new_rows])

# def run_xgb(x_train, x_valid, y_train, y_valid):
def run_xgb(x_train, x_test, x_label):

    # x_train = pd.concat([pos_train, neg_train]) #Concat positive and negative
    # y_train = (np.zeros(len(pos_train)) + 1).tolist() + np.zeros(len(neg_train)).tolist() #Putting in 1 and 0

    # x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242)

    # Set our parameters for xgboost
    params = {}
    params['objective'] = 'binary:logistic'
    params['eval_metric'] = 'logloss'
    params['eta'] = 0.05
    params['max_depth'] = 6
    params['silent'] = 1

    d_train = xgb.DMatrix(x_train, label=x_label)
    d_test = xgb.DMatrix(x_test)

    watchlist = [(d_train, 'train')]

    bst = xgb.train(params, d_train, 100, watchlist, early_stopping_rounds=50, verbose_eval=50)

    p_test = bst.predict(d_test)

    # xgb.plot_importance(bst)
    # pyplot.show()

    return p_test

def run_tsne(pos_train, neg_train, x_test_feat):

    x_train = pd.concat([pos_train, neg_train]) #Concat positive and negative
    y_train = (np.zeros(len(pos_train)) + 1).tolist() + np.zeros(len(neg_train)).tolist() #Putting in 1 and 0

    x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242)

    df_subsampled = x_train[0:3000]
    X = MinMaxScaler().fit_transform(df_subsampled[['z_len1', 'z_len2', 'z_words1', 'z_words2', 'word_match']])
    # y = y_train['is_duplicate'].values

    tsne = TSNE(
        n_components=3,
        init='random', # pca
        random_state=101,
        method='barnes_hut',
        n_iter=200,
        verbose=2,
        angle=0.5
    ).fit_transform(X)

    trace1 = go.Scatter3d(
        x=tsne[:,0],
        y=tsne[:,1],
        z=tsne[:,2],
        mode='markers',
        marker=dict(
            sizemode='diameter',
            color = y_train,
            colorscale = 'Portland',
            colorbar = dict(title = 'duplicate'),
            line=dict(color='rgb(255, 255, 255)'),
            opacity=0.75
        )
    )

    data=[trace1]
    layout=dict(height=800, width=800, title='3d embedding with engineered features')
    fig=dict(data=data, layout=layout)
    py.plot(data, filename='3d_bubble')

def validate(training):

    training_res = training.pop("is_duplicate")
    x_train, x_valid, y_train, y_valid = train_test_split(training, training_res, test_size=0.2, random_state=4242, stratify = training_res)

    return(x_train, x_valid, y_train, y_valid)

def real_testing(df_with_qs, gen_filename):

    # Required for initial setup!
    # dataframe_modified = df_with_qs.progress_apply(basic_nlp, axis = 1)
    old_filename = './old/' + gen_filename
    # dataframe_modified.to_csv(old_filename, index = False)

    dataframe_modified = pd.read_csv(old_filename).fillna("")

    dataframe_modified["neighbor_intersection"] = df_with_qs.apply(neighbor_intersection, axis = 1)

    # q1_second_degree_freq = dataframe.apply(get_q1_second_degree_freq, axis = 1)
    # q2_second_degree_freq = dataframe.apply(get_q2_second_degree_freq, axis = 1)
    # dataframe_modified["second_degree_avg"] = (q1_second_degree_freq + q2_second_degree_freq)/2
    # dataframe_modified["second_degree_diff"] = abs(q1_second_degree_freq - q2_second_degree_freq)
    # dataframe_modified["second_degree_intersection"] = dataframe.apply(second_degree_intersection, axis = 1)
    # dataframe_modified["separation"] = dataframe.progress_apply(initialize_bfs, axis = 1)

    new_filename = "./new/" + gen_filename
    dataframe_modified.to_csv(new_filename, index=False)
    # %reset_selective dataframe_modified

def pred_n_submit(x_train, x_label, test_filename, test_id_df, res_filename):

    x_test = pd.read_csv(test_filename).fillna("")

    res_1 = run_xgb(x_train, x_test, x_label)
    sub = pd.DataFrame()

    sub['test_id'] = test_id_df['test_id']
    sub['is_duplicate'] = res_1

    sub.to_csv(res_filename, index=False)
    # %reset_selective -f x_test_1

if __name__ == '__main__':

    df_train = pd.read_csv('./train.csv').fillna("")
    df_test = pd.read_csv('./test.csv').fillna("")

    train_qs = pd.Series(df_train['question1'].tolist() + df_train['question2'].tolist()).astype(str)
    test_qs = pd.Series(df_test['question1'].tolist() + df_test['question2'].tolist()).astype(str)
    qs = pd.concat([train_qs, test_qs])

    # tfidf = TfidfVectorizer(max_features = 256, stop_words='english', ngram_range=(1, 1))
    # tfidf.fit_transform(train_qs[0:2500])

    #Load up the Weights Dictionary
    # words = (" ".join(qs)).lower().split()
    # counts = Counter(words)
    # weights = {word: get_inverse_freq(1/(10000 + int(count)), count) for word, count in counts.items()}
    # with open('word_weights.pickle', 'wb') as handle:
    #     pickle.dump(weights, handle)
    with open('word_weights.pickle', 'rb') as handle:
        weights = pickle.load(handle)

    stops = set(stopwords.words("english"))

    #Load up the hashtable
    # hash_table = {}
    # df_train.apply(generate_hash_freq, axis = 1)
    # df_test.apply(generate_hash_freq, axis = 1)
    # with open('hash_table.pickle', 'wb') as handle:
    #     pickle.dump(hash_table, handle)
    with open('hash_table.pickle', 'rb') as handle:
        hash_table = pickle.load(handle)

    # pos_hash_table = {}
    # df_train.apply(generate_duplicate_freq, axis = 1)
    # with open('pos_hash_table.pickle', 'wb') as handle:
    #     pickle.dump(pos_hash_table, handle)

    # with open('pos_hash_table.pickle', 'rb') as handle:
    #     pos_hash_table = pickle.load(handle)

    # hash_table_ne = {}
    # df_train.apply(generate_ne_freq, axis = 1)
    # df_test.apply(generate_ne_freq, axis = 1)
    # with open('hash_table_ne.pickle', 'wb') as handle:
    #     pickle.dump(hash_table_ne, handle)
    # with open('hash_table_ne.pickle', 'rb') as handle:
    #     hash_table_ne = pickle.load(handle)

    #Load up the graph!
    # graph = {}
    # df_train.apply(generate_graph_table, axis = 1)
    # df_test.progress_apply(generate_graph_table, axis = 1)
    # with open('graph.pickle', 'wb') as handle:
    #     pickle.dump(graph, handle)
    with open('graph.pickle', 'rb') as handle:
        graph = pickle.load(handle)

    # pos_graph = {}
    # df_train.apply(generate_positive_graph, axis = 1)
    # with open('pos_graph.pickle', 'wb') as handle:
    #     pickle.dump(pos_graph, handle)
    # with open('pos_graph.pickle', 'rb') as handle:
    #     pos_graph = pickle.load(handle)

    #Augment Test Data
    # new_df_test = df_test
    # df_test.apply(augment_test, axis = 1)

    #Validation
    # x_train, x_test, y_train, y_valid = validate(df_train)
    # x_train_feat = x_train.apply(basic_nlp, axis = 1)
    # x_test_feat = x_test.apply(basic_nlp, axis = 1)
    # res = run_xgb(x_train_feat, x_test_feat, y_train, y_valid)

    real_testing(df_train, 'x_train.csv')
    real_testing(df_test[0:390000], 'x_test_1.csv')
    real_testing(df_test[390000:780000], 'x_test_2.csv')
    real_testing(df_test[780000:1170000], 'x_test_3.csv')
    real_testing(df_test[1170000:1560000], 'x_test_4.csv')
    real_testing(df_test[1560000:1950000], 'x_test_5.csv')
    real_testing(df_test[1950000:], 'x_test_6.csv')

    #Finally!
    x_train = pd.read_csv('./new/x_train.csv').fillna("")
    x_label = df_train.is_duplicate
    oversample_label = 0

    if oversample_label == 1:
        x_train["is_duplicate"] = df_train.is_duplicate
        x_train = oversample(x_train)

        x_label = x_train.pop("is_duplicate")
        # res_oversampled = run_xgb(x_train_oversampled, x_test, x_label_oversampled)
        # submit(res_oversampled)

    pred_n_submit(x_train, x_label, './new/x_test_1.csv', df_test[0:390000], './res_files/res_1.csv')
    pred_n_submit(x_train, x_label, './new/x_test_2.csv', df_test[390000:780000], './res_files/res_2.csv')
    pred_n_submit(x_train, x_label, './new/x_test_3.csv', df_test[780000:1170000], './res_files/res_3.csv')
    pred_n_submit(x_train, x_label, './new/x_test_4.csv', df_test[1170000:1560000], './res_files/res_4.csv')
    pred_n_submit(x_train, x_label, './new/x_test_5.csv', df_test[1560000:1950000], './res_files/res_5.csv')
    pred_n_submit(x_train, x_label, './new/x_test_6.csv', df_test[1950000:], './res_files/res_6.csv')

    res_1 = pd.read_csv('./res_files/res_1.csv').fillna("")
    res_2 = pd.read_csv('./res_files/res_2.csv').fillna("")
    res_3 = pd.read_csv('./res_files/res_3.csv').fillna("")
    res_4 = pd.read_csv('./res_files/res_4.csv').fillna("")
    res_5 = pd.read_csv('./res_files/res_5.csv').fillna("")
    res_6 = pd.read_csv('./res_files/res_6.csv').fillna("")

    res = pd.concat([res_1, res_2, res_3, res_4, res_5, res_6])
    res.to_csv("jus_tryin.csv", index = False)

    #After submitting paste files in ./new to ./old -- Building upon the already generated features


import csv
import os
from datetime import date, datetime
import yfinance as yf
import pandas as pd
import glob
from RedditScrape import create_data_folder

"""
This program scrapes the current stock price of all the stocks listed in a
csv file generated by RedditScrape and creates another csv file in directory
StockPrices
"""

def get_current_price(symbol):
    """
    returns the price of the ticker
    :param symbol: string, the name of the stock
    :return: float, the value of the stock
    """
    try:
        ticker = yf.Ticker(symbol)
        todays_data = ticker.history(period='1d')
        return todays_data['Close'][0]
    except:
        return None

def get_current_volume(symbol):
    """
    returns the volume of the stock
    :param symbol: string, the name of the stock
    :return: int, the volume of the stock
    """
    try:
        ticker = yf.Ticker(symbol)
        todays_data = ticker.history(period='1d')
        return todays_data['Volume'][0]
    except:
        return None

def create_name(subdir, name, symb):
    """
    Returns the filepath which the csv file will be saved to
    :param subdir: string, The subdirectory
    :param name: string, the name of the file
    :param symb: boolean, checks if this is for accessing the data file
    :return: string, the path and name of the file
    """
    count = 1
    here = os.path.dirname(os.path.realpath(__file__))
    present = datetime.now()
    pres = present.strftime("%H-%M-%S")
    if symb:
        return os.path.join(here, subdir) + "\\"
    filename = f"{date.today()}-time-{pres}-{name}"
    filepath = os.path.join(here, subdir, filename)
    while os.path.exists(filepath):
        if count == 1:
            filepath = filepath[:filepath.index(".")] + f"({count})" + \
                       filepath[filepath.index("."):len(filepath)]
        else:
            filepath = filepath.replace(f"({count - 1})", f"({count})")
        count += 1
    return filepath

def create_symbols(subdir) -> [str]:
    """
    Returns a list of all the symbols in the most recent file in the directory
    :param subdir: string, the name of the subdirectory
    :return: list, the list of all the symbols
    """
    list_of_files = glob.glob(f'{create_name(subdir, None, True)}*.csv')
    latest_file = max(list_of_files, key=os.path.getctime)
    col_list = ["Stock Name"]
    df = pd.read_csv(latest_file, usecols=col_list)
    list_stock = df[col_list[0]].tolist()
    return list_stock

def create_stock_csv() -> None:
    """
    Creates a .csv with all the data inside the class
    :return: None
    """
    subdir = create_data_folder("StockPrices", "Stonk")
    name = "Stock.csv"
    fold = create_data_folder("ScrapedData", "Scraped")
    filepath = create_name(subdir, name, False)
    with open(filepath, "w", newline="") as file:
        writer = csv.writer(file)
        writer.writerow(
            ["Stock Name", "Price", "Volume"])
        for stk in create_symbols(fold):
            writer.writerow(
                [stk, get_current_price(stk), get_current_volume(stk)])


##UTLILISE LE FILTRE DE CANNY
import time
from mpu9250_jmdev.registers import *
from mpu9250_jmdev.mpu_9250 import MPU9250
import VL53L0X

import picamera
import datetime

import numpy as np
from numpy import asarray
import matplotlib.pyplot as plt
from PIL import Image as img
import math as m
import cv2
#import cv2 as cv

def capture(name,x=3280, y=1845): #accède à la caméra, prend une capture et la sauvegarde sous le nom "name" : le "name doit inclure le formar .png ou jpg
    camera = picamera.PiCamera()
    camera.resolution = (x,y)
    camera.capture(name)
    imBrute = img.open(name)
    return (imBrute)

def traitementImageCanny(imBrute): #application du filtre de canny : Binarisation de l'image, application d'un filtre gradient pour détection de contours

    imTraitement = cv2.imread(imBrute.filename)  # Read image

    #Setting parameter values
    t_lower = 80  # Lower Threshold
    t_upper = 110  # Upper threshold

    # Applying the Canny Edge filter
    edge = cv2.Canny(imTraitement, t_lower, t_upper)
    imTraitee=img.fromarray(edge)
    imTraitee.show()
    return (imTraitee)

def constructionMatriceAccumulatrice(imTraitee,resolutionAngulaire = 1, resolutionDistance=1): #construction de la matrice accumulatrice pour la détection de droite
    theta = [k for k in range(0,int(180/resolutionAngulaire),1)]
    distanceMax=int((((3280/resolutionDistance)**2)+((1845/resolutionDistance)**2))**(1/2))+1 #longueur maximale de rho dnas le plan de hough, conditionné par la diagonale
    matriceAccumulatrice = np.zeros((int(180/resolutionAngulaire),distanceMax)) #matrice qui permettra le dessin du plan de hough
    coord= []
    for x in range(int(3280/resolutionDistance)) :
        for y in range(int(1845/resolutionDistance)):
            if imTraitee.getpixel((x*resolutionDistance,y*resolutionDistance))==255:
                coord.append((x*resolutionDistance,y*resolutionDistance))
    #récupéré les coordonnées des points d'intérêt

    for a in coord :
        for T in theta:
            T=T*resolutionAngulaire
            x=a[0]
            y=a[1]
            r=((x/resolutionDistance)*m.cos((T*2*m.pi)/360))+((y/resolutionDistance)*m.sin((T*2*m.pi)/360))
            matriceAccumulatrice[int(T/resolutionAngulaire)][int(r)] +=1
    return ([matriceAccumulatrice, imTraitee])

def reconstructionDroite(matriceAccumulatrice,imBrute,resolutionAngulaire,resolutionDistance, nbligne):
    largeurX = len(matriceAccumulatrice[0])
    longueurY = len(matriceAccumulatrice[0][0])
    planDeHough = img.new("L",size=(largeurX,longueurY))
    for x in range(largeurX):
        for y in range (longueurY):
            planDeHough.putpixel((x,y),int(matriceAccumulatrice[0][x,y]))
    planDeHough.save("plan_de_hough.jpg")
    #on montre le plan de hough
    plt.imshow(planDeHough,cmap="binary_r")
    plt.xlabel("theta")
    plt.ylabel("rho")
    plt.suptitle("plan de hough")
    plt.show()

    #recherche du max
    max_index = np.unravel_index(matriceAccumulatrice[0].argmax(), matriceAccumulatrice[0].shape) #trouve l'index d'un max de la matrice
    m=matriceAccumulatrice[0][max_index]
    listMax=[max_index]
    valeurmax=matriceAccumulatrice[0][max_index]
    matriceAccumulatrice[0][max_index]=0
    nb=1;
    while (nb<nbligne):
        max_index = np.unravel_index(matriceAccumulatrice[0].argmax(), matriceAccumulatrice[0].shape)
        while ((((max_index[0]-listMax[0][0])**2)+((max_index[1]-listMax[0][1])**2))**(1/2) <=20):
            matriceAccumulatrice[0][max_index]=0
            max_index = np.unravel_index(matriceAccumulatrice[0].argmax(), matriceAccumulatrice[0].shape)

        listMax.append(max_index)
        nb+=1

    print("index du max et valeur :",max_index,valeurmax)

    #coloriage des max
    for z in listMax:
        x=z[0]
        y=z[1]
        for i in range(-1,2):
            for j in range(-1,2):
                if x+i >=0 and y+j >=0 and x+i<largeurX and y+j <longueurY  :
                    planDeHough.putpixel((x+i,y+j),190)
    plt.imshow(planDeHough,cmap="binary_r")
    plt.xlabel("theta")
    plt.ylabel("rho")
    plt.suptitle("plan de hough avec repérage des max")
    plt.show()

    print("ci joint la liste des maximas globaux de la matrice accumulatrice :")
    print(listMax)

    distanceMax=int(((3280**2)+(1845**2))**(1/2)+1) #diagonale maximale
    print ("distance max = ", distanceMax)
    stockEquation=[]
    for w in range(len(listMax)):
        thetaL=listMax[w][0]*resolutionAngulaire
        rhoL=listMax[w][1]*resolutionDistance
        print("thetaL=",thetaL,"rhoL =",rhoL)
        while (rhoL>=distanceMax/2) :
            print(rhoL, "  avant")
            rhoL =rhoL - distanceMax
            print(rhoL, "  après")

        #equation de la forme ax + b
        if (thetaL!=0):
            a1=-np.cos(thetaL*2*np.pi/360)/np.sin(thetaL*2*np.pi/360)
            b1=rhoL/np.sin(thetaL*2*np.pi/360)
            x=[k for k in range(3280)]
            y=[((a1*x[k])+b1) for k in x]
            print ("équation trouvée : ","y = ",a1,"x + ",b1)
            stockEquation.append([x,y])
        else :
            x=[rhoL for k in range(3280)]
            y=[k for k in range(3280)]
            stockEquation.append([x,y])

        for k in range(len(x)):
            if y[k]>=0 and y[k]<1845:
                imBrute.putpixel((x[k],int(y[k])),150)
            if y[k]>=0 and y[k]+1<1845:
                imBrute.putpixel((x[k],int(y[k]+1)),150)
            if y[k]>=0 and y[k]-1<1845:
                imBrute.putpixel((x[k],int(y[k]-1)),150)
            if y[k]>=0 and y[k]<1845 and x[k]+1<3280:
                imBrute.putpixel((x[k]+1,int(y[k])),150)
            if y[k]>=0 and y[k]<1845 and x[k]-1>=0:
                imBrute.putpixel((x[k]-1,int(y[k])),150)

        #tracage du centre
        for u in range(-1,2):
            for i in range(-10,11):
                if 1640+u+i >= 0 and 1640 + u +i <3280 and 922+u+i>=0 and 922+u+i<1845:
                    imBrute.putpixel((1640+u+i,922+u),150)
                    imBrute.putpixel((1640+u,922+i+u),150)
    imBrute.show()

    return stockEquation

def Horizon(imageBrute,resolutionAngulaire,resolutionDistance, nbligne):
    stockEquation=reconstructionDroite(constructionMatriceAccumulatrice(traitementImageCanny(imageBrute),resolutionAngulaire,resolutionDistance),imageBrute,resolutionAngulaire,resolutionDistance, nbligne)
    return stockEquation

### cercle dont on connait le rayon

def coordCercleRayonEstime(a,b,rayonEstime,matrice,resolutionDistance, resolutionAngulaireCercleInter):#a et b coordonnnées réelles de son centre en pixels, resol détermine le nombre de points du cercle
    theta=0
    pasAngulaire=(2*m.pi)/resolutionAngulaireCercleInter
    while(theta<2*m.pi):
        x=int(rayonEstime*m.cos(theta)+a)
        y=int(rayonEstime*m.sin(theta)+b)
        theta+=pasAngulaire
        if ((x<3280) and (y<1845) and (x>=0) and (y>=0)):
            matrice[int(x/resolutionDistance)][int(y/resolutionDistance)]+=1

def constructionMatriceAccumulatriceCercleConnu(imTraitee,rayonEstime, resolutionAngulaire = 1, resolutionDistance=1, resolutionAngulaireCercleInter=32):

    matriceAccumulatriceCercleConnu = np.zeros((int(3280/resolutionAngulaire)+1,int(1845/resolutionDistance)+1)) #matrice qui permettra le dessin du plan de hough
    coord= [] #rassemble les points d'intérêt, i.e ceux qui sont un contour
    for x in range(int(3280/resolutionDistance)) :
        for y in range(int(1845/resolutionDistance)):
            if imTraitee.getpixel((x*resolutionDistance,y*resolutionDistance))==255:
                coordCercleRayonEstime(x*resolutionDistance,y*resolutionDistance,rayonEstime,matriceAccumulatriceCercleConnu,resolutionDistance,resolutionAngulaireCercleInter)
    #récupéré les coordonnées des points d'intérêt, en pixels de l'image, sans les considérations de résolution
    return matriceAccumulatriceCercleConnu

def reconstructionCercle(matriceAccumulatriceCercleConnu,imBrute,rayonEstime,resolutionAngulaire=1,resolutionDistance=1,resolutionAngulaireCercleInter=32):
    largeurX = len(matriceAccumulatriceCercleConnu)
    longueurY = len(matriceAccumulatriceCercleConnu[0])
    planDeHoughCercle = img.new("L",size=(largeurX,longueurY))
    for x in range(largeurX):
        for y in range (longueurY):
            planDeHoughCercle.putpixel((x,y),int(matriceAccumulatriceCercleConnu[x,y]))
    planDeHoughCercle.save("plan_de_hough.jpg")
    #on montre le plan de hough
    plt.imshow(planDeHoughCercle,cmap="binary_r")
    plt.xlabel("x")
    plt.ylabel("y")
    plt.suptitle("plan de hough")
    plt.show()

    #recherche du max
    max_index = np.unravel_index(matriceAccumulatriceCercleConnu.argmax(), matriceAccumulatriceCercleConnu.shape) #trouve l'index d'un max de la matrice
    m=matriceAccumulatriceCercleConnu[max_index]
    matriceAccumulatriceCercleConnu[max_index]=0
    listMax=[max_index]
    print("index du max et valeur :",max_index,m)
    while (matriceAccumulatriceCercleConnu[np.unravel_index(matriceAccumulatriceCercleConnu.argmax(), matriceAccumulatriceCercleConnu.shape)]==m):
        listMax.append(np.unravel_index(matriceAccumulatriceCercleConnu.argmax(), matriceAccumulatriceCercleConnu.shape))
        matriceAccumulatriceCercleConnu[np.unravel_index(matriceAccumulatriceCercleConnu.argmax(), matriceAccumulatriceCercleConnu.shape)]=0

    #coloriage des max
    for z in listMax:
        x=z[0]
        y=z[1]
        for i in range(-2,3):
            for j in range(-2,3):
                if (x+i<int(3280/resolutionDistance)) and (x+i >0) and (y+i<int(1845/resolutionDistance)) and (y+i >0):
                    planDeHoughCercle.putpixel((x+i,y+j),210)
    plt.imshow(planDeHoughCercle,cmap="binary_r")
    plt.xlabel("x")
    plt.ylabel("y")
    plt.suptitle("plan de hough avec repérage du max")
    plt.show()

    print("ci joint la liste des maximas globaux de la matrice accumulatrice :")
    print(listMax)
    aCentre=listMax[0][0]*resolutionDistance
    bCentre=listMax[0][1]*resolutionDistance

    print("aCentre=",aCentre,"bCentre =",bCentre)

    #determination des coordonnees du cercle determiné pour le tracer sur l'image brute
    theta=0
    coordCercle=[]
    pasAngulaire=(2*np.pi)/(resolutionAngulaireCercleInter*20) #le coef multiplicatif est là pour traçer plus de points du cercle
    while (theta<2*np.pi):
        coordCercle.append((int(rayonEstime*np.cos(theta)+aCentre),int(rayonEstime*np.sin(theta)+bCentre)))
        theta+=pasAngulaire

    print("quelques coordonnées du cercle et la longueur de la liste : ", len(coordCercle),"  ", coordCercle[0],"   ", coordCercle[-1])
    for c in coordCercle:
        for k in range(-3,4):
            for j in range(-3,4):
                if ((c[0]+k<3280) and (c[1]+j<1845) and (c[0]+k>=0) and (c[1]+j>=0)):
                    imBrute.putpixel((c[0]+k,c[1]+j),120)
    for k in range(-3,4):
            for j in range(-3,4):
                if ((aCentre+k<3280) and (bCentre+j<1845) and (aCentre+k>=0) and (bCentre+j>=0)):
                    imBrute.putpixel((aCentre+k,bCentre+j),120)
    plt.imshow(imBrute)
    plt.xlabel("x")
    plt.ylabel("y")
    plt.suptitle("image brute avec le cercle trouvé")
    plt.show()

def reconnaissanceCercleRayonConnu(imageBrute,rayonEstime,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter):
    reconstructionCercle(constructionMatriceAccumulatriceCercleConnu(traitementImageCanny(imageBrute), rayonEstime,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter),imageBrute, rayonEstime,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter)

###Cercle de rayon inconnu

def coordCercleRayonR(a,b,r,matrice,pasDeRayon,resolutionDistance, resolutionAngulaireCercleInter):#a et b coordonnnées réelles de son centre en pixels, resol détermine le nombre de points du cercle
    theta=0
    pasAngulaire=(2*m.pi)/resolutionAngulaireCercleInter
    while(theta<2*m.pi):
        x=int(r*m.cos(theta)+a)
        y=int(r*m.sin(theta)+b)
        theta+=pasAngulaire
        if ((x<3280) and (y<1845) and (x>=0) and (y>=0)):
            matrice[int(r/pasDeRayon)][int(x/resolutionDistance)][int(y/resolutionDistance)]+=50

def traitementImageCannyGradient(imBrute, stockEquation=[]):
    imTraitement = cv2.imread(imBrute.filename)  # Read image

    #Setting parameter values
    t_lower = 60  # Lower Threshold
    t_upper = 110  # Upper threshold

    # Applying the Canny Edge filter
    imTraitee = cv2.Canny(imTraitement, t_lower, t_upper)

    #effaçage des droites trouvées pour ne pas perturber la détection de cercle sur l'image NB de canny

    for k in range(len(stockEquation)):
        x=stockEquation[k][0]
        y=stockEquation[k][1]
        for l in range(len(x)):
            for i in range(-15,16):
                for j in range(-15,16):
                    if round(x[l])+i < 3280 and round(x[l])+i >=0:
                        if round(y[l])+j>=0 and round(y[l])+j <1845:
                            if imTraitee[round(y[l])+j][round(x[l])+i] >10:
                                imTraitee[round(y[l])+j][round(x[l])+i]=0

    apercu=img.fromarray(imTraitee)
    apercu.show()
    return (imTraitee)

def constructionMatriceAccumulatriceCercleInconnu(imTraitee,pasDeRayon=10, rayonMini=25 ,resolutionAngulaire = 1, resolutionDistance=1, resolutionAngulaireCercleInter=32):
    #matrices de convolution
    Gx=np.array(([-1,0,1],[-2,0,2],[-1,0,1]))
    Gy=np.array(([-1,-2,-1],[0,0,0],[1,2,1]))
    distanceMax=round(1845/2)
    #on considère que le rayon du cercle que l'on vera a un rayon minimal et un rayon maximal
    matriceAccumulatriceCercleInconnu=np.zeros((((distanceMax-rayonMini)//pasDeRayon)+2,round(1845/resolutionDistance)+2,round(3280/resolutionDistance)+1))
    CXY=[] #liste des coordonnées des points du cercle de la matrice accumulatrice
    for x in range(round(3280/resolutionDistance)) :
        for y in range(round(1845/resolutionDistance)):
            if imTraitee[y*resolutionDistance,x*resolutionDistance]==255:
                matriceAccumulatriceCercleInconnu[len(matriceAccumulatriceCercleInconnu)-1,y,x]=1
                CXY.append([[x,y]])#faux pixels
    for i in range(len(CXY)):
        x=CXY[i][0][0]*resolutionDistance #conversion en vrais pixels
        y=CXY[i][0][1]*resolutionDistance #conversion en vrais pixels
        if x>=1 and y>=1 and x<3279 and y<1844 :
            A=np.array(([imTraitee[y-1,x-1],imTraitee[y-1,x],imTraitee[y-1,x+1]],[imTraitee[y,x-1],imTraitee[y,x],imTraitee[y,x+1]],[imTraitee[y+1,x-1],imTraitee[y+1,x],imTraitee[y+1,x+1]])) #matrice 3x3 autour du point du cercle considéré
            CXY[i].append(m.atan2((((Gy*A).sum())/9),(((Gx*A).sum())/9)))
            for r in range(rayonMini,distanceMax,pasDeRayon):
                theta=CXY[i][1]*360/(2*np.pi)
                U=np.arange(theta-25,theta+25,1)
                x=CXY[i][0][0]
                y=CXY[i][0][1]
                for T in U:
                    a=round(r*m.cos(T*2*np.pi/360)+x)
                    b=round(r*m.sin(T*2*np.pi/360)+y)
                    if (a>=0) and (a<round(3280/resolutionDistance)) and (b>=0) and (b<round(1845/resolutionDistance)):
                        matriceAccumulatriceCercleInconnu[((r-rayonMini)//pasDeRayon),b,a]+=1
                    a=round(-r*m.cos(-T*2*np.pi/360)+x)
                    b=round(-r*m.sin(-T*2*np.pi/360)+y)
                    if (a>=0) and (a<round(3280/resolutionDistance)) and (b>=0) and (b<round(1845/resolutionDistance)):
                        matriceAccumulatriceCercleInconnu[((r-rayonMini)//pasDeRayon),b,a]+=1
    plt.imshow(matriceAccumulatriceCercleInconnu[len(matriceAccumulatriceCercleInconnu)-1])
    plt.show()
    return matriceAccumulatriceCercleInconnu

def reconstructionCercleInconnu(matriceAccumulatriceCercleInconnu,imBrute,pasDeRayon=10,rayonMini=25,resolutionAngulaire=1,resolutionDistance=1,resolutionAngulaireCercleInter=32):
    #recherche du max
    max_index = np.unravel_index(matriceAccumulatriceCercleInconnu.argmax(), matriceAccumulatriceCercleInconnu.shape) #trouve l'index d'un max de la matrice
    m=matriceAccumulatriceCercleInconnu[max_index]
    matriceAccumulatriceCercleInconnu[max_index]=0
    listMax=[max_index]
    print("index du max et valeur :",max_index,m)
    while (matriceAccumulatriceCercleInconnu[np.unravel_index(matriceAccumulatriceCercleInconnu.argmax(), matriceAccumulatriceCercleInconnu.shape)]==m):
        listMax.append(np.unravel_index(matriceAccumulatriceCercleInconnu.argmax(), matriceAccumulatriceCercleInconnu.shape))
        matriceAccumulatriceCercleInconnu[np.unravel_index(matriceAccumulatriceCercleInconnu.argmax(), matriceAccumulatriceCercleInconnu.shape)]=0
    print("longueur liste des maxima : ", len(listMax), "      coordonnées : ", listMax)

    bCentre=listMax[0][1]*resolutionDistance
    aCentre=listMax[0][2]*resolutionDistance
    rayon=((listMax[0][0]*pasDeRayon)+rayonMini)*resolutionDistance

    plt.imshow(matriceAccumulatriceCercleInconnu[listMax[0][0]])
    plt.show()

    for i in range(-3,4):
        for j in range(-3,4):
            imBrute.putpixel((int(aCentre)+i,int(bCentre)+j),180)

    print("aCentre=",aCentre,"   bCentre =",bCentre,"   rayon =",rayon)

    #determination des coordonnees du cercle determiné pour le tracer sur l'image brute
    theta=0
    coordCercle=[]
    pasAngulaire=(2*np.pi)/(resolutionAngulaireCercleInter*20) #le coef multiplicatif est là pour traçer plus de points du cercle
    while (theta<2*np.pi):
        coordCercle.append((int(rayon*np.cos(theta)+aCentre),int(rayon*np.sin(theta)+bCentre)))
        theta+=pasAngulaire

    for c in coordCercle:
        for k in range(-3,4):
            for j in range(-3,4):
                if ((c[0]+k<3280) and (c[1]+j<1845) and (c[0]+k>=0) and (c[1]+j>=0)):
                    imBrute.putpixel((c[0]+k,c[1]+j),120)
    for k in range(-3,4):
            for j in range(-3,4):
                if ((aCentre+k<3280) and (bCentre+j<1845) and (aCentre+k>=0) and (bCentre+j>=0)):
                    imBrute.putpixel((aCentre+k,bCentre+j),120)
    #centre du cercle
    for u in range(-1,2):
            for i in range(-10,11):
                if aCentre+u+i >= 0 and aCentre + u +i <3280 and bCentre+u+i>=0 and bCentre+u+i<1845:
                    imBrute.putpixel((aCentre+u+i,bCentre+u),150)
                    imBrute.putpixel((aCentre+u,bCentre+i+u),150)

    plt.imshow(imBrute)
    plt.xlabel("x")
    plt.ylabel("y")
    plt.suptitle("image brute avec le cercle trouvé")
    plt.show()
    return  [imBrute,aCentre,bCentre]

def reconnaissanceCercleRayonInconnu(imageBrute,pasDeRayon,rayonMini,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter):
    reconstructionCercleInconnu(constructionMatriceAccumulatriceCercleInconnu(traitementImageCannyGradient(imageBrute), pasDeRayon,rayonMini,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter),imageBrute, pasDeRayon,rayonMini,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter)

def MatriceDistances (distance, angle=0, FOVX=62, FOVY=49, resol=20):
    matriceDist=np.zeros((2,int(1845/resol)+1,int(3280/resol) +1))
    for x in range(0,1640,resol):
        for y in range(0,922,resol):
            thetaX=(FOVX/2)*(x/1640)
            thetaY=(FOVY/2)*(y/922)
            dx=m.tan(thetaX*m.pi/180)*distance
            dy=m.tan(thetaY*m.pi/180)*distance
            matriceDist[0][(int(1845/(2*resol)))+int(y/resol)][(int(3280/(2*resol)))+int(x/resol)]=dx
            matriceDist[0][(int(1845/(2*resol)))+int(y/resol)][(int(3280/(2*resol)))-int(x/resol)]=dx
            matriceDist[0][(int(1845/(2*resol)))-int(y/resol)][(int(3280/(2*resol)))+int(x/resol)]=dx
            matriceDist[0][(int(1845/(2*resol)))-int(y/resol)][(int(3280/(2*resol)))-int(x/resol)]=dx
            matriceDist[1][(int(1845/(2*resol)))+int(y/resol)][(int(3280/(2*resol)))+int(x/resol)]=dy
            matriceDist[1][(int(1845/(2*resol)))+int(y/resol)][(int(3280/(2*resol)))-int(x/resol)]=dy
            matriceDist[1][(int(1845/(2*resol)))-int(y/resol)][(int(3280/(2*resol)))+int(x/resol)]=dy
            matriceDist[1][(int(1845/(2*resol)))-int(y/resol)][(int(3280/(2*resol)))-int(x/resol)]=dy
    return matriceDist

def total(imBrute,pasDeRayon,rayonMini,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter, nblignes): #combine la détection de lignes et de cercle

    stockEquation=Horizon(imBrute,resolutionAngulaire,resolutionDistance,nblignes)
    [imBrute,aCentre,bCentre]=reconstructionCercleInconnu(constructionMatriceAccumulatriceCercleInconnu(traitementImageCannyGradient(imBrute, stockEquation), pasDeRayon,rayonMini,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter),imBrute, pasDeRayon,rayonMini,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter)
    distance=21
    A=MatriceDistances(distance)
    print("coordonnées par rapport au centre de l image (centre optique) : ", A[0][round(bCentre/20)][round(aCentre/20)], A[1][round(bCentre/20)][round(aCentre/20)])
#total(img.open("blablabla.jpg"),5,50,1,4,128,2)

def totalCam(nom,pasDeRayon,rayonMini,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter, nblignes): #combine la prise de photo via caméra, l'utilisation des capteurs, et la détection de lignes et de cercle
    [imBrute,distance,angleHorizontal]=captureEcranData(nom)
    stockEquation=Horizon(imBrute,resolutionAngulaire,resolutionDistance,nblignes)
    [imBrute,aCentre,bCentre]=reconstructionCercleInconnu(constructionMatriceAccumulatriceCercleInconnu(traitementImageCannyGradient(imBrute, stockEquation), pasDeRayon,rayonMini,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter),imBrute, pasDeRayon,rayonMini,resolutionAngulaire,resolutionDistance,resolutionAngulaireCercleInter)
    A=MatriceDistances(distance)
    print("coordonnées par rapport au centre de l image (centre optique) : ", A[0][round(bCentre/20)][round(aCentre/20)], A[1][round(bCentre/20)][round(aCentre/20)])

# def a(): #algorithme déjà proposé par uen bibliothère python : très rapide, mais très sensible au bruit, imprécis, non robuste, inexact, approximatif
    image = cv2.imread("cercle.jpg")
    image=cv2.cvtColor(image,cv2.COLOR_GRAY2BGR)
    #th,image=cv2.threshold(image,90,255,cv2.THRESH_BINARY)
    th,image = cv2.threshold(image, 128,192,cv2.THRESH_OTSU)
    image = cv2.medianBlur(image,5)
    cv2.namedWindow("detected circles", cv2.WINDOW_NORMAL)
    cv2.resizeWindow("detected circles", 300, 700)
    cv2.imshow('detected circles',image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
    cimg = cv2.cvtColor(image,cv2.COLOR_GRAY2BGR)
    circles = cv2.HoughCircles(image,cv2.HOUGH_GRADIENT,1,20, param1=50,param2=30,minRadius=10,maxRadius=500)
    circles = np.uint16(np.around(circles))
    for i in circles[0]:
        # draw the outer circle
        cv2.circle(cimg,(i[0],i[1]),i[2],(0,255,0),2)
        # draw the center of the circle
        cv2.circle(cimg,(i[0],i[1]),2,(0,0,255),3)

    cv2.namedWindow("detected circles", cv2.WINDOW_NORMAL)
    cv2.resizeWindow("detected circles", 300, 700)
    cv2.imshow('detected circles',cimg)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

    ### Concerne les capteurs de distance et gyromètre

### gyromètre
mpu = MPU9250(
    address_ak=AK8963_ADDRESS,
    address_mpu_master=MPU9050_ADDRESS_68, # Master has 0x68 Address
    address_mpu_slave=MPU9050_ADDRESS_68, # Slave has 0x68 Address
    bus=1,
    gfs=GFS_1000,
    afs=AFS_8G,
    mfs=AK8963_BIT_16,
    mode=AK8963_MODE_C100HZ)

mpu.configure() # Apply the settings to the registers.

def captureAngle(mpu=mpu):
    accelerationX = mpu.readAccelerometerMaster()[0]
    accelerationY = mpu.readAccelerometerMaster()[1]
    accelerationZ = mpu.readAccelerometerMaster()[2]
    pitch = 180 * m.atan (accelerationX/m.sqrt(accelerationY*accelerationY + accelerationZ*accelerationZ))/m.pi;
    print("angle : ", pitch)

    ### Capteur de distance

#Create a VL53L0X object

def captureDistance():
    tof = VL53L0X.VL53L0X(i2c_bus=1,i2c_address=0x29)

    tof.open()
    # Start ranging
    tof.start_ranging(VL53L0X.Vl53l0xAccuracyMode.BETTER)

    distance = tof.get_distance()
    if distance > 0:
        print("%d mm, %d cm, %d" % (distance, (distance/10),1))

    tof.stop_ranging()
    tof.close()
    return distance/10 #renvoie la valeur en centimètres

###capture d'écran + données !

def captureEcranData(name):
    imBrute = capture(name,x=3280, y=1845)
    angleHorizontal=captureAngle()
    distance=captureDistance()
    print("angleHorizontal = ", angleHorizontal, "    distance = ", distance )
    return [imBrute,distance,angleHorizontal]


import logging

from rest_framework import generics
from django.shortcuts import render

from .models import Stops, Calendar, Trips, StopTimes
from .serializers import StopsSerializer, TripsSerializer, ArrivalTimesSerializer
from . import forms

logging.config.dictConfig({
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'console': {
            'format': '%(name)-12s %(levelname)-8s %(message)s'
        },
        'file': {
            'format': '%(asctime)s %(name)-12s %(levelname)-8s %(message)s'
        }
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'formatter': 'console'
        },
        'file': {
            'level': 'INFO',
            'class': 'logging.FileHandler',
            'formatter': 'file',
            'filename': 'transport/tmp/debug.log'
        }
    },
    'loggers': {
        '': {
            'level': 'DEBUG',
            'handlers': ['console', 'file'],
        },
        'django.request': {
            'level': 'DEBUG',
            'handlers': ['console', 'file'],
            'propagate': False,
        }
    }
})
logger = logging.getLogger(__name__)

def index(request):
  station_names_list = Stops.objects.filter(
        stop_id__in=StopTimes.objects.all().values_list(
        'stop_id', flat=True).distinct()).values_list('stop_name', 'stop_id').order_by('stop_id')

  station_names_dict = dict(station_names_list)
  return render(request, 'transport/index.html', {'arrival_form': forms.ArrivalTimesForm, 'alltrips_form': forms.AllTripsForm, 'station_names_dict': station_names_dict})

class ArrivalTimesOfStop(generics.ListAPIView):
  """
  This view displays the arrival times of trains for a given stop -all of them-, by station_id through the api of by station name through the form.
  """
  serializer_class = ArrivalTimesSerializer

  def get_queryset(self, *args, **kwargs):
    if self.request.query_params.get('station'):
      stop_name = self.request.query_params.get('station')
      queryset = StopTimes.objects.filter(stop_id=(Stops.objects.filter(stop_name=stop_name).values('stop_id'))[0]['stop_id'])
      log = [stop_name, 'through the form']

    elif self.kwargs['stop']:
      stop = self.kwargs['stop']
      queryset = StopTimes.objects.filter(stop_id=f'{stop}')
      log = [stop, 'through the url path']

    logger.info(f"All arrival times of stop: '{log[0]}' requested {log[1]}.")
    return queryset

class TripsOnGivenDay(generics.ListAPIView):
  """
  This ApiView provides a list of trips (service id, trip id and route direction) that run on a given day. It serves both the api and the form requests.
  """
  serializer_class = TripsSerializer

  def get_queryset(self, *args, **kwargs):

    if self.request.query_params.get('days'):
      day = self.request.query_params.get('days')
      log = 'through the form'
    elif self.kwargs['day']:
      day = self.kwargs['day']
      log = 'through the url path'

    queryset = Trips.objects.filter(service_id__in=Calendar.objects.filter(**{f"{day}":'1'})).values('service', 'trip_id', 'route_direction')
    logger.info(f'All trips on {day} requested {log}.')
    return queryset


import datetime
import re

class AnvlParseException(Exception):
    pass

_pattern1 = re.compile("[%:\r\n]")
_pattern2 = re.compile("[%\r\n]")
_pattern3 = re.compile("%([0-9a-fA-F][0-9a-fA-F])?")

def _encode(pattern, s):
    return pattern.sub(lambda c: f"%{ord(c.group(0)):02X}", s)

def _encodeLabel(s):
    return _encode(_pattern1, s)

def _encodeValue(s):
    if not isinstance(s, str):
        s = str(s)
    return _encode(_pattern2, s)

def _decodeRewriter(m):
    if len(m.group(0)) == 3:
        return chr(int(m.group(0)[1:], 16))
    else:
        raise AnvlParseException("percent-decode error")

def _decode(s):
    return _pattern3.sub(_decodeRewriter, s)

def formatPair(label, value, indent=""):
    """Format a label and value into an ANVL element
    """
    if isinstance(value, datetime.datetime):
        value = value.strftime("%Y.%m.%d_%H:%M:%S")
    elif isinstance(value, dict):
        value = format(value, indent=indent+4*" ")
        return f"{_encodeLabel(label)}:\n{value}"
    return f"{indent}{_encodeLabel(label)}: {_encodeValue(value)}\n"

def format(d, indent=""):
    """Format a dictionary into an ANVL string

    Labels and values are suitably percent-encoded.
    """
    return "".join(formatPair(k, v, indent=indent) for k, v in list(d.items()))

def parse(s):
    """Parse an ANVL string and returns a dictionary

    Labels and values are percent-decoded. Raises AnvlParseException
    (defined in this module).
    """
    d = {}
    k = None
    # We avoid splitlines here to avoid splitting on other weirdo
    # Unicode characters that count as line breaks.
    for l in re.split("\r\n?|\n", s):
        if len(l) == 0:
            k = None
        elif l[0] == "#":
            pass
        elif l[0].isspace():
            if k is None:
                raise AnvlParseException("no previous label for continuation line")
            ll = _decode(l).strip()
            if ll != "":
                if d[k] == "":
                    d[k] = ll
                else:
                    d[k] += " " + ll
        else:
            if ":" not in l:
                raise AnvlParseException("no colon in line")
            k, v = [_decode(w).strip() for w in l.split(":", 1)]
            if len(k) == 0:
                raise AnvlParseException("empty label")
            if k in d:
                raise AnvlParseException("repeated label")
            d[k] = v
    return d

def parseConcatenate(s):
    """Alternate version of 'parse' that concatenates repeated label values and
    separates them by semicolons. For example, the input string:

      a: b
      a: c

    produces the output dictionary:

      { "a": "b ; c" }
    """
    d = {}
    k = None
    # We avoid splitlines here to avoid splitting on other weirdo
    # Unicode characters that count as line breaks.
    for l in re.split("\r\n?|\n", s):
        if len(l) == 0:
            k = None
        elif l[0] == "#":
            pass
        elif l[0].isspace():
            if k is None:
                raise AnvlParseException("no previous label for continuation line")
            ll = _decode(l).strip()
            if ll != "":
                if d[k] == "":
                    d[k] = ll
                else:
                    d[k] += " " + ll
        else:
            if ":" not in l:
                raise AnvlParseException("no colon in line")
            k, v = [_decode(w).strip() for w in l.split(":", 1)]
            if len(k) == 0:
                raise AnvlParseException("empty label")
            if k in d and d[k] != "":
                if v != "":
                    d[k] += " ; " + v
            else:
                d[k] = v
    return d


import wx
from wx.lib import newevent

from amulet_map_editor.api.opengl.camera import (
    CameraMovedEvent,
    EVT_CAMERA_MOVED,
    ProjectionChangedEvent,
    EVT_PROJECTION_CHANGED,
    SpeedChangedEvent,
    EVT_SPEED_CHANGED,
)

from amulet_map_editor.api.opengl.events import (
    PreDrawEvent,
    EVT_PRE_DRAW,
    DrawEvent,
    EVT_DRAW,
    PostDrawEvent,
    EVT_POST_DRAW,
)

from amulet_map_editor.api.wx.util.button_input import (
    InputPressEvent,
    InputHeldEvent,
    InputReleaseEvent,
    EVT_INPUT_PRESS,
    EVT_INPUT_HELD,
    EVT_INPUT_RELEASE,
)

from .selection import SelectionChangeEvent, EVT_SELECTION_CHANGE

DimensionChangeEvent, EVT_DIMENSION_CHANGE = newevent.NewEvent()

# the active tool changed
_ToolChangeEventType = wx.NewEventType()
EVT_TOOL_CHANGE = wx.PyEventBinder(_ToolChangeEventType)

class ToolChangeEvent(wx.PyEvent):
    """Run when the camera has moved or rotated."""

    def __init__(self, tool: str, state=None):
        wx.PyEvent.__init__(self, eventType=_ToolChangeEventType)
        self.tool = tool
        self.state = state

UndoEvent, EVT_UNDO = newevent.NewEvent()
RedoEvent, EVT_REDO = newevent.NewEvent()
CreateUndoEvent, EVT_CREATE_UNDO = newevent.NewEvent()
SaveEvent, EVT_SAVE = newevent.NewEvent()
EditCloseEvent, EVT_EDIT_CLOSE = newevent.NewEvent()


import torch
import torch.nn as nn
from torch.nn.utils import weight_norm
import torch.nn.functional as F
import numpy as np
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class ScaledDotProductAttention(nn.Module):
    def __init__(self):
        super(ScaledDotProductAttention, self).__init__()

    def forward(self, Q, K, V):
        B, n_heads, len1, len2, d_k = Q.shape
        scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k)

        attn = nn.Softmax(dim=-1)(scores)
        context = torch.matmul(attn,
                               V)  # [batch_size, n_heads, T(Spatial) or N(Temporal), N(Spatial) or T(Temporal), d_k]]
        return context

class SMultiHeadAttention(nn.Module):
    def __init__(self, embed_size, heads):
        super(SMultiHeadAttention, self).__init__()

        self.embed_size = embed_size
        self.heads = heads
        self.head_dim = embed_size // heads
        self.W_V = nn.Linear(self.embed_size, self.head_dim * self.heads, bias=False)
        self.W_K = nn.Linear(self.embed_size, self.head_dim * self.heads, bias=False)
        self.W_Q = nn.Linear(self.embed_size, self.head_dim * self.heads, bias=False)
        self.fc_out = nn.Linear(heads * self.head_dim, embed_size)

    def forward(self, input_Q, input_K, input_V):
        '''
        input_Q: [batch_size, N, T, C]
        input_K: [batch_size, N, T, C]
        input_V: [batch_size, N, T, C]
        attn_mask: [batch_size, seq_len, seq_len]
        '''
        B, N, T, C = input_Q.shape
        # [B, N, T, C] --> [B, N, T, h * d_k] --> [B, N, T, h, d_k] --> [B, h, T, N, d_k]
        Q = self.W_Q(input_Q).view(B, N, T, self.heads, self.head_dim).transpose(1, 3)  # Q: [B, h, T, N, d_k]
        K = self.W_K(input_K).view(B, N, T, self.heads, self.head_dim).transpose(1, 3)  # K: [B, h, T, N, d_k]
        V = self.W_V(input_V).view(B, N, T, self.heads, self.head_dim).transpose(1, 3)  # V: [B, h, T, N, d_k]
        context = ScaledDotProductAttention()(Q, K, V)  # [B, h, T, N, d_k]
        context = context.permute(0, 3, 2, 1, 4)  # [B, N, T, h, d_k]
        context = context.reshape(B, N, T, self.heads * self.head_dim)  # [B, N, T, C]
        output = self.fc_out(context)  # [batch_size, len_q, d_model]
        return output

class TMultiHeadAttention(nn.Module):
    def __init__(self, embed_size, heads):
        super(TMultiHeadAttention, self).__init__()

        self.embed_size = embed_size
        self.heads = heads
        self.head_dim = embed_size // heads

        assert (
                self.head_dim * heads == embed_size
        ), "Embedding size needs to be divisible by heads"

        # 用Linear来做投影矩阵
        # 但这里如果是多头的话，是不是需要声明多个矩阵？？？

        self.W_V = nn.Linear(self.embed_size, self.head_dim * self.heads, bias=False)
        self.W_K = nn.Linear(self.embed_size, self.head_dim * self.heads, bias=False)
        self.W_Q = nn.Linear(self.embed_size, self.head_dim * self.heads, bias=False)
        self.fc_out = nn.Linear(heads * self.head_dim, embed_size)

    def forward(self, input_Q, input_K, input_V):
        '''
        input_Q: [batch_size, N, T, C]
        input_K: [batch_size, N, T, C]
        input_V: [batch_size, N, T, C]
        attn_mask: [batch_size, seq_len, seq_len]
        '''
        B, N, T, C = input_Q.shape
        # [B, N, T, C] --> [B, N, T, h * d_k] --> [B, N, T, h, d_k] --> [B, h, N, T, d_k]
        Q = self.W_Q(input_Q).view(B, N, T, self.heads, self.head_dim).permute(0, 3, 1, 2, 4)  # Q: [B, h, N, T, d_k]
        K = self.W_K(input_K).view(B, N, T, self.heads, self.head_dim).permute(0, 3, 1, 2, 4)  # K: [B, h, N, T, d_k]
        V = self.W_V(input_V).view(B, N, T, self.heads, self.head_dim).permute(0, 3, 1, 2, 4)  # V: [B, h, N, T, d_k]

        # attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size, n_heads, seq_len, seq_len]

        # context: [batch_size, n_heads, len_q, d_v], attn: [batch_size, n_heads, len_q, len_k]
        context = ScaledDotProductAttention()(Q, K, V)  # [B, h, N, T, d_k]
        context = context.permute(0, 2, 3, 1, 4)  # [B, N, T, h, d_k]
        context = context.reshape(B, N, T, self.heads * self.head_dim)  # [B, N, T, C]
        # context = context.transpose(1, 2).reshape(batch_size, -1, n_heads * d_v) # context: [batch_size, len_q, n_heads * d_v]
        output = self.fc_out(context)  # [batch_size, len_q, d_model]
        return output

class Chomp1d(nn.Module):
    def __init__(self, chomp_size):
        super(Chomp1d, self).__init__()
        self.chomp_size = chomp_size

    def forward(self, x):
        return x[:, :, :-self.chomp_size].contiguous()

class TemporalBlock(nn.Module):
    def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2):
        super(TemporalBlock, self).__init__()
        self.conv1 = weight_norm(nn.Conv1d(n_inputs, n_outputs, kernel_size,
                                           stride=stride, padding=padding, dilation=dilation))
        self.chomp1 = Chomp1d(padding)
        self.relu1 = nn.ReLU()
        self.dropout1 = nn.Dropout(dropout)

        self.conv2 = weight_norm(nn.Conv1d(n_outputs, n_outputs, kernel_size,
                                           stride=stride, padding=padding, dilation=dilation))
        self.chomp2 = Chomp1d(padding)
        self.relu2 = nn.ReLU()
        self.dropout2 = nn.Dropout(dropout)

        self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1,
                                 self.conv2, self.chomp2, self.relu2, self.dropout2)
        self.downsample = nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None
        self.relu = nn.ReLU()
        self.init_weights()

    def init_weights(self):
        self.conv1.weight.data.normal_(0, 0.01)
        self.conv2.weight.data.normal_(0, 0.01)
        if self.downsample is not None:
            self.downsample.weight.data.normal_(0, 0.01)

    def forward(self, x):
        out = self.net(x)
        res = x if self.downsample is None else self.downsample(x)
        return self.relu(out + res)

class TemporalConvNet(nn.Module):
    def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2):
        super(TemporalConvNet, self).__init__()
        layers = []
        num_levels = len(num_channels)
        for i in range(num_levels):
            dilation_size = 2 ** i
            in_channels = num_inputs if i == 0 else num_channels[i-1]
            out_channels = num_channels[i]
            layers += [TemporalBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,
                                     padding=(kernel_size-1) * dilation_size, dropout=dropout)]

        self.network = nn.Sequential(*layers)

    def forward(self, x):
        return self.network(x)

class nconv(nn.Module):
    def __init__(self):
        super(nconv,self).__init__()

    def forward(self,x, A):
        x = torch.einsum('ncvl,vw->ncwl',(x,A))
        return x.contiguous()

class linear(nn.Module):
    def __init__(self,c_in,c_out):
        super(linear,self).__init__()
        self.mlp = torch.nn.Conv2d(c_in, c_out, kernel_size=(1, 1), padding=(0,0), stride=(1,1), bias=True)

    def forward(self,x):
        return self.mlp(x)

class gcn(nn.Module):
    def __init__(self,c_in,c_out,dropout,support_len=3,order=2):
        super(gcn,self).__init__()
        self.nconv = nconv()
        c_in = (order*support_len+1)*c_in
        self.mlp = linear(c_in,c_out)
        self.dropout = dropout
        self.order = order

    def forward(self,x,support):
        x = x.permute(0, 3, 1, 2)
        out = [x]
        #support = [support]
        for a in support:
            x1 = self.nconv(x,a)
            out.append(x1)
            for k in range(2, self.order + 1):
                x2 = self.nconv(x1,a)
                out.append(x2)
                x1 = x2

        h = torch.cat(out,dim=1)
        h = self.mlp(h)
        h = F.dropout(h, self.dropout, training=self.training)
        return h.permute(0,2,3,1)

class STBlk(nn.Module):
    def __init__(self,STadj,embedsize,nodes,num_channels,heads,dropout=0):
        super(STBlk, self).__init__()
        self.nodes = nodes
        self.tcn1 = TemporalConvNet(num_inputs=embedsize, num_channels=num_channels, kernel_size=3,dropout=dropout)
        self.tcn2 = TemporalConvNet(num_inputs=embedsize, num_channels=num_channels, kernel_size=2, dropout=dropout)
        self.pos_embed_1 = nn.Parameter(torch.zeros(1, nodes, 12, embedsize))

        self.STadj = STadj
        self.gcn1 = gcn(embedsize,embedsize,dropout,support_len=2)
        self.gcn2 = gcn(embedsize, embedsize, dropout, support_len=2)
        self.gcn3 = gcn(embedsize, embedsize, dropout, support_len=1)

        self.SAT = SMultiHeadAttention(embedsize, heads)

        self.TAT = TMultiHeadAttention(embedsize, heads)

        self.relu = nn.ReLU()

        self.feed_forward = nn.Sequential(
            nn.Linear(embedsize, 4 * embedsize),
            nn.ReLU(),
            nn.Linear(4 * embedsize, embedsize),
        )

        self.feed_forward1 = nn.Sequential(
            nn.Linear(embedsize, 4 * embedsize),
            nn.ReLU(),
            nn.Linear(4 * embedsize, embedsize),
        )

        self.norm1 = nn.LayerNorm(embedsize)
        self.norm2 = nn.LayerNorm(embedsize)

        self.norm3 = nn.LayerNorm(embedsize)
        self.norm4 = nn.LayerNorm(embedsize)

        self.norm5 = nn.LayerNorm(embedsize)
        self.norm6 = nn.LayerNorm(embedsize)

        self.tcngate = nn.Sequential(
            nn.Linear(embedsize, embedsize),
            nn.Sigmoid()
        )
        self.gcngate = nn.Sequential(
            nn.Linear(embedsize, embedsize),
            nn.Sigmoid()
        )

        self.dropout = nn.Dropout(dropout)

        self.fc1 = nn.Sequential(
            nn.Linear(embedsize,embedsize),
            nn.Linear(embedsize, embedsize),
        )

        self.fc2 = nn.Sequential(
            nn.Linear(embedsize,embedsize),
            nn.Linear(embedsize, embedsize),
        )

    def forward(self,x_T,x_S,Adj):

        res_T = x_T
        res_S = x_S
        x_tcn = x_T
        x_gcn = x_S

        B,N,T,H = x_T.shape

        x_T_Q = x_S
        x_T_K = x_T
        x_T_V = x_T

        x_S_Q = x_T
        x_S_K = x_S
        x_S_V = x_S

        ########################时间

        x_T = self.TAT(x_T_Q,x_T_K,x_T_V)
        x_tcn = x_tcn.reshape(B,N*T,H).permute(0,2,1)
        x_tcn =torch.tanh(self.tcn1(x_tcn).permute(0,2,1).reshape(B,N,T,H))*torch.sigmoid(self.tcn2(x_tcn).permute(0,2,1).reshape(B,N,T,H))
        x_T = x_T*self.tcngate(x_tcn)
        x_T = self.dropout(self.norm1(x_T+res_T))
        x_feed = self.feed_forward(x_T)
        x_T = self.dropout(self.norm2(x_feed+x_T))

        ########################空间

        x_S = self.SAT(x_S_Q,x_S_K,x_S_V)

        x_gcn_F = torch.tanh(self.gcn1(x_gcn, Adj[0:2])) * torch.sigmoid(self.gcn2(x_gcn, Adj[0:2]))
        x_gcn_D = self.gcn3(x_gcn, Adj[2:3])
        g = torch.sigmoid(self.fc1(x_gcn_F) + self.fc2(x_gcn_D))
        x_gcn = g * x_gcn_F + (1 - g) * x_gcn_D

        x_S = x_S*self.gcngate(x_gcn)
        x_S= self.dropout(self.norm3(x_S+res_S))
        x_feed = self.feed_forward1(x_S)
        x_S = self.dropout(self.norm4(x_feed+x_S))

        return x_T,x_S

class STGHTN(nn.Module):
    def __init__(self,STadj,embedsize,nodes,num_channels,heads,dropout=0):
        super(STGHTN, self).__init__()
        self.nodes = nodes
        self.STadj = STadj
        self.ST1 = STBlk(STadj, embedsize, nodes, num_channels, heads, dropout)
        self.ST2 = STBlk(STadj, embedsize, nodes, num_channels, heads, dropout)
        self.ST3 = STBlk(STadj, embedsize, nodes, num_channels, heads, dropout)
        self.ST4 = STBlk(STadj, embedsize, nodes, num_channels, heads, dropout)

        self.conv1 = nn.Conv2d(1, embedsize, 1)
        self.conv2 = nn.Conv2d(12, 12, 1)
        self.conv3 = nn.Conv2d(embedsize, 1, 1)
        self.norm1 = nn.LayerNorm(embedsize)
        self.norm2 = nn.LayerNorm(embedsize)
        self.norm3 = nn.LayerNorm(embedsize)
        # self.norm2 = nn.LayerNorm(12)
        self.relu = nn.ReLU()

        self.fs = nn.Linear(embedsize, embedsize)
        self.fg = nn.Linear(embedsize, embedsize)

        self.fs1 = nn.Linear(embedsize, embedsize)
        self.fg1 = nn.Linear(embedsize, embedsize)

        self.pos_embed = nn.Parameter(torch.zeros(1, nodes, 12, embedsize), requires_grad=True)
        self.Linear1 = nn.Sequential(
            nn.Linear(embedsize,embedsize),
            nn.Linear(embedsize,1),
            nn.LeakyReLU(),
        )
        self.Linear2 = nn.Sequential(
        nn.Linear(embedsize, embedsize),
        nn.Linear(embedsize, embedsize),
        nn.ReLU(),
        )
        self.Linear3 = nn.Sequential(
        nn.Linear(embedsize, embedsize//2),
        nn.Linear(embedsize//2, 1),
        nn.ReLU(),
        )

        self.feed_forward = nn.Sequential(
            nn.Linear(embedsize, 4 * embedsize),
            nn.ReLU(),
            nn.Linear(4 * embedsize, embedsize),
        )

        self.feed_forward1 = nn.Sequential(
            nn.Linear(embedsize, 4 * embedsize),
            nn.ReLU(),
            nn.Linear(4 * embedsize, embedsize),
        )
        m, p, n = torch.svd(STadj[1])
        initemb1 = torch.mm(m[:, :10], torch.diag(p[:10] ** 0.5))
        initemb2 = torch.mm(torch.diag(p[:10] ** 0.5), n[:, :10].t())
        self.nodevec1 = nn.Parameter(initemb1, requires_grad=True).to(device)
        self.nodevec2 = nn.Parameter(initemb2, requires_grad=True).to(device)
    def forward(self, x):  #B T N
        x = x.permute(0, 2, 1).unsqueeze(1)
        input = self.conv1(x)
        input = input.permute(0, 2, 3, 1)
        input = input+self.pos_embed
        T_SKIP = 0
        S_SKIP = 0
        B,N,T,H = input.shape

        adp = F.softmax(F.relu(torch.mm(self.nodevec1, self.nodevec2)), dim=1)*(self.STadj[0]+self.STadj[1])
        #print(adp)
        Adj = self.STadj+[adp]
        #Adj = self.STadj
        new_Adj = []
        for i in range(len(Adj)):
            if i<2:
                A_H = Adj[i]+torch.eye(self.nodes).to(device)
            else:
                A_H = Adj[i]
            D = (A_H.sum(-1)**-0.5)
            D[torch.isinf(D)]=0.
            D = torch.diag_embed(D)
            A = torch.matmul(torch.matmul(D, A_H), D)
            new_Adj.append(A)
        #print(new_Adj[-1])
        ###############################################
        x_T, x_S = self.ST1(input, input,new_Adj)
        T_SKIP = x_T + T_SKIP
        S_SKIP = x_S + S_SKIP
        x_T, x_S = self.ST2(x_T, x_S,new_Adj)
        T_SKIP = x_T + T_SKIP
        S_SKIP = x_S + S_SKIP
        x_T, x_S = self.ST3(x_T, x_S,new_Adj)
        T_SKIP = x_T + T_SKIP
        S_SKIP = x_S + S_SKIP
        x_T, x_S = self.ST4(x_T, x_S,new_Adj)
        T_SKIP = x_T + T_SKIP
        S_SKIP = x_S + S_SKIP

        feed_T = self.feed_forward(T_SKIP)
        T_SKIP = self.norm1(feed_T+T_SKIP)

        feed_S = self.feed_forward1(S_SKIP)
        S_SKIP = self.norm2(feed_S+S_SKIP)
        x = T_SKIP + S_SKIP
        #################################################
        out = x.permute(0, 2, 1, 3)
        out = self.relu(self.conv2(out))
        out = out.permute(0, 3, 2, 1)
        out = self.conv3(out)
        out = out.squeeze(1)
        # print(out)
        return out.permute(0, 2, 1)


import os
from datetime import datetime

from flask import Flask, redirect, render_template, request, send_from_directory, url_for, flash
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from flask_wtf.csrf import CSRFProtect

import threading
import sys

from werkzeug.utils import secure_filename
#
import subprocess
from subprocess import Popen
from subprocess import PIPE
from itertools import islice
from threading import Thread
from queue import Queue, Empty

WORKERS=1
UPLOAD_FOLDER = '/tmp/uploads'
ALLOWED_EXTENSIONS = {'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'}

JOB_LOG=""
#app = Flask(__name__)

app = Flask(__name__, static_folder='static')
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.JOB_LOG=""
app.COMMANDS=""

csrf = CSRFProtect(app)

# WEBSITE_HOSTNAME exists only in production environment
if 'WEBSITE_HOSTNAME' not in os.environ:
    # local development, where we'll use environment variables
    print("Loading config.development and environment variables from .env file.")
    app.config.from_object('azureproject.development')
else:
    # production
    print("Loading config.production.")
    app.config.from_object('azureproject.production')

app.config.update(
    SQLALCHEMY_DATABASE_URI=app.config.get('DATABASE_URI'),
    SQLALCHEMY_TRACK_MODIFICATIONS=False,
)

# Initialize the database connection
db = SQLAlchemy(app)

# Enable Flask-Migrate commands "flask db init/migrate/upgrade" to work
migrate = Migrate(app, db)

# The import must be done after db initialization due to circular import issue
from models import Person, Company

@app.route('/', methods=['GET'])
def index():
    print('Request for index page received')
    all_records = Company.query.all()
    return render_template('index.html', records=all_records)

@app.route('/<int:id>', methods=['GET'])
def details(id):
    master = Company.query.where(Company.id == id).first()
    detail_rows = Person.query.where(Person.company_id == id)
    return render_template('details.html', master=master, detail_rows=detail_rows)

@app.route('/create', methods=['GET'])
def create_master():
    print('Request for add master record page received')
    return render_template('create_master.html')

@app.route('/add', methods=['POST'])
@csrf.exempt
def add_master():
    try:
        name = request.values.get('name')
        street_address = request.values.get('street_address')
        description = request.values.get('description')

    except (KeyError):
        # Redisplay the question voting form.
        return render_template('create_master.html', {
            'error_message': "You must include all fields",
        })
    else:
        record  = Company()
        record.name = name
        record.street_address=street_address
        record.description = description

        db.session.add(record)
        db.session.commit()
        return redirect(url_for('details', id= record.id  ))

@app.route('/editdetails/<int:id>', methods=['GET','POST' ])
@csrf.exempt
def edit_details(id):
    row_details=""
    if request.method == 'POST':
        try:
            name = request.values.get('name')
            email = request.values.get('email')
            street_address = request.values.get('street_address')
            phone = request.values.get('phone')
            company_id = request.values.get('company_id')
        except (KeyError):
            #Redisplay the question voting form.
            return render_template('add_details.html', {
                'error_message': "Error adding record",
            })
        else:
            record = Person.query.filter_by(id=id).one()

            record.name = name
            record.email = email
            record.street_address=street_address
            record.phone = phone
            record.modification_date=datetime.now()
            record.company_id=company_id

            db.session.add(record)
            db.session.commit()
            return redirect(url_for('details', id=company_id))
    else:
            row_details  = Person.query.filter_by(id=id).one()
    return render_template('edit_details.html', row_details=row_details)

@app.route('/deletedetails/<int:id>', methods=['GET'])
@csrf.exempt
def delete_details(id):
    try:
        person = Person.query.filter_by(id=id).one()
        print(person)
        company_id=person.company_id
        print("company_id:")
        print(company_id)
        db.session.delete(person)
        db.session.commit()
    except (KeyError):
        #Redisplay the question voting form.
        return render_template('add_details.html', {
            'error_message': "Error adding record",
        })
    return redirect(url_for('details', id=company_id))

@app.route('/details/<int:id>', methods=['POST'])
@csrf.exempt
def add_details(id):
    try:
        name = request.values.get('name')
        email = request.values.get('email')
        street_address = request.values.get('street_address')
        phone = request.values.get('phone')
    except (KeyError):
        #Redisplay the question voting form.
        return render_template('add_details.html', {
            'error_message': "Error adding record",
        })
    else:
        record = Person()
        record.company_id = id
        record.name = name
        record.email = email
        record.street_address=street_address
        record.phone = phone
        record.modification_date=datetime.now()

        db.session.add(record)
        db.session.commit()

    return redirect(url_for('details', id=id))

@app.context_processor
def utility_processor():
    def star_rating(id):
        reviews = Measure.query.where(Measure.cm_id == id)

        ratings = []
        review_count = 0
        for review in reviews:
            ratings += [review.rating]
            review_count += 1

        avg_rating=0
        stars_percent=0
        #avg_rating = sum(ratings) / len(ratings) if ratings else 0
        #stars_percent = round((avg_rating / 5.0) * 100) if review_count > 0 else 0
        return {'avg_rating': avg_rating, 'review_count': review_count, 'stars_percent': stars_percent}

    return dict(star_rating=star_rating)

@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'),
                               'favicon.ico', mimetype='image/vnd.microsoft.icon')

####
def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route('/upload', methods=['GET', 'POST'])
@csrf.exempt
def upload_file():
    if request.method == 'POST':
        # check if the post request has the file part
        if 'file' not in request.files:
            flash('No file part')
            return redirect(request.url)
        file = request.files['file']
        # If the user does not select a file, the browser submits an
        # empty file without a filename.
        if file.filename == '':
            flash('No selected file')
            return redirect(request.url)
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            #f = request.files['file'] puts the uploaded file (in the request) to a var ("f"). Then
            content=file.read()
            return content #redirect(url_for('download_file', name=filename))
    return '''
    <!doctype html>
    <title>Upload new File</title>

Upload new File

    <form method=post enctype=multipart/form-data>
      <input type=file name=file>
      <input type=submit value=Upload>
    </form>
    '''

from flask import send_from_directory

@app.route('/uploads/<name>')
@csrf.exempt
def download_file(name):
    return send_from_directory(app.config["UPLOAD_FOLDER"], name)

if __name__ == '__main__':
    app.run()


from pytube import YouTube

class Youtube_Downloader():
    def __init__(self):
        self.output_path = ''

    def functionA(self, url, filename):
        yt = YouTube(url)
        streams = yt.streams.filter(subtype='mp4').all()
        mp4 = yt.streams.first()
        mp4.download(self.output_path, filename)
        # YouTube(url).streams.first().download()

    def functionB(self, url, filename, index):
        try:
            yt = YouTube(url)
            print(yt.streams.get_by_itag(index).download(self.output_path, filename))
        except:
            self.functionB(url, filename, 22)

if __name__ == '__main__':
    yd = Youtube_Downloader()
    url = 'https://www.youtube.com/watch?v=O7U2PiENOec'
    file = 'Watermelon Smash Challenge play with Gaby Alex and Mommy'
    file = file.replace(':', '_')
    file = file.replace(' ', '_')
    print(file)
    yd.functionB(url, file, 37)


import torch.nn as nn
import numpy as np
def conv_block (in_channels, out_channels):
    return nn.Sequential(
        nn.Conv2d(in_channels, out_channels, 3, padding=1),
        nn.BatchNorm2d(out_channels),
        nn.ReLU(inplace=False),
        nn.MaxPool2d(2)
    )

class ProtoNet(nn.Module):
    def __init__(self, x_dim = 1, hid_dim = 64, z_dim = 64):
        super(ProtoNet, self).__init__()
        self.encoder=nn.Sequential(
            conv_block(x_dim, hid_dim),
            conv_block(hid_dim, hid_dim),
            conv_block(hid_dim,hid_dim),
            conv_block(hid_dim, z_dim)
        )
    def forward(self, x):
        x = x.transpose(1, 3)
        print(x.shape)
        x = self.encoder(x)
        return x.view(x.size(0), -1)


import sys
from cx_Freeze import setup, Executable

# GUI applications require a different base on Windows (the default is for a
# console application).
base = None
if sys.platform == "win32":
    base = "Win32GUI"

setup(
    name = "Análisis Numérico",
    version = "0.0.1",
    author = "Josué Amaya - @c3rberuss_",
    author_email = "c3rberuss@gmail.com",
    description = ("Implementación de los algoritmos de los método numéricos siguientes: Método de Bisección, Método de punto fijo"),
    keywords = "analisis metodo numerico biseccion punto fijo",
    executables = [Executable("main.pyw", base=base)]
    )


from pqdict import PQDict
import matplotlib.pyplot as plt
import networkx as nx

def dijkstra(G, start, end=None):
    inf = float('inf')
    D = {start: 0}          # mapping of nodes to their dist from start
    Q = PQDict(D)           # priority queue for tracking min shortest path
    P = {}                  # mapping of nodes to their direct predecessors
    U = set(G.keys())       # unexplored nodes

    while U:                                    # nodes yet to explore
        (v, d) = Q.popitem()                    # node w/ min dist d on frontier
        D[v] = d                                # est dijkstra greedy score
        U.remove(v)                             # remove from unexplored
        if v == end: break

        # now consider the edges from v with an unexplored head -
        # we may need to update the dist of unexplored successors
        for w in G[v]:                          # successors to v
            if w in U:                          # then w is a frontier node
                d = int(D[v]) + int(G[v][w])              # dgs: dist of start -> v -> w
                if d < Q.get(w, inf):
                    Q[w] = d                    # set/update dgs
                    P[w] = v                    # set/update predecessor

    return D, P

def shortest_path(G, start, end):
    dist, pred = dijkstra(G, start, end)
    v = end
    path = [v]
    path2 = []
    while v != start:
        path2.append((pred[v],v))
        v = pred[v]
        path.append(v)
    path.reverse()
    path2.reverse()
    return path, path2

def make_graph(G, shortPath):
    DG = nx.DiGraph()
    adj = G
    e = [(u, v, {'weight': d}) for u, nbrs in adj.items()
         for v, d in nbrs.items()]
    DG.update(edges = e, nodes=adj)
    pos=nx.spring_layout(DG)
    edge_labels = {(n1,n2): DG[n1][n2]['weight'] for (n1,n2) in DG.edges()}
    red_edges = shortPath
    edge_colors = ['black' if not edge in red_edges else 'red' for edge in DG.edges()]
    options = {
        'node_color': 'blue',
        'node_size': 500,
        'width': 2,
        'arrowstyle': '-|>',
        'arrowsize': 12,
        'edge_color' : edge_colors
    }
    nx.draw_networkx_edge_labels(DG,pos,edge_labels=edge_labels)
    nx.draw_networkx(DG, pos, arrows=True, **options)
    plt.show()
    # G = {}
    #
    # with open(filename) as file:
    #     for row in file:
    #         r = row.strip().split('\t')
    #         label = r.pop(0)
    #         neighbors = {v: int(length) for v, length in [e.split(',') for e in r]}
    #         G[label] = neighbors
    #
    # return G