import re
import os
import sys
import json
import spacy
import string
import logging
import argparse

from nltk.corpus import stopwords

ATTN_WINDOW_RANGE = 0

def _process_strings(line,
                     lang_nlp,
                     get_lemmas,
                     get_pos,
                     remove_stopwords,
                     replace_stopwords,
                     get_maps):

    """ Helper function for obtaining various word representations """

    # strip, replace special tokens
    orig_line = line
    line = line.strip()
    line = re.sub(r''', '\'', line.strip())
    line = re.sub(r'"', '\"', line.strip())
    # Tokenize etc.
    line_nlp = lang_nlp(line)
    spacy_tokens = [elem.text for elem in line_nlp]
    spacy_tokens_lower = [elem.text.lower() for elem in line_nlp]
    spacy_lemmas = None
    spacy_pos = None
    if get_lemmas:
        spacy_lemmas = list()
        for elem in line_nlp:
            if elem.lemma_ == '-PRON-' or elem.lemma_.isdigit():
                spacy_lemmas.append(elem.lower_)
            else:
                spacy_lemmas.append(elem.lemma_.lower().strip())
    if get_pos:
        spacy_pos = [elem.pos_ for elem in line_nlp]

    # Generate a mapping between whitespace tokens and SpaCy tokens
    ws_tokens = orig_line.strip().split()
    ws_tokens_lower = orig_line.strip().lower().split()
    ws_to_spacy_map = dict()
    spacy_to_ws_map = dict()
    if get_maps:
        ws_loc = 0
        ws_tok = ws_tokens[ws_loc]

        for spacy_loc, spacy_tok in enumerate(spacy_tokens):
            while True:
                # Map whitespace tokens to be identical to spacy tokens
                ws_tok = re.sub(r''', '\'', ws_tok)
                ws_tok = re.sub(r'"', '\"', ws_tok)

                if spacy_tok == ws_tok or spacy_tok in ws_tok:
                    # Terminate
                    if ws_loc >= len(ws_tokens):
                        break

                    # Extend maps
                    if not ws_to_spacy_map.get(ws_loc, None):
                        ws_to_spacy_map[ws_loc] = list()
                    ws_to_spacy_map[ws_loc].append(spacy_loc)
                    if not spacy_to_ws_map.get(spacy_loc, None):
                        spacy_to_ws_map[spacy_loc] = list()
                    spacy_to_ws_map[spacy_loc].append(ws_loc)

                    # Move pointer
                    if spacy_tok == ws_tok:
                        ws_loc += 1
                        if ws_loc < len(ws_tokens):
                            ws_tok = ws_tokens[ws_loc]
                    else:
                        ws_tok = ws_tok[len(spacy_tok):]
                    break
                else:
                    ws_loc += 1

        # Assert full coverage of whitespace and SpaCy token sequences by the mapping
        ws_covered = sorted(list(ws_to_spacy_map.keys()))
        spacy_covered = sorted(list(set(list([val for val_list in ws_to_spacy_map.values() for val in val_list]))))
        assert ws_covered == [n for n in range(len(ws_tokens))], \
            'WS-SpaCy mapping does not cover all whitespace tokens: {}; number of tokens: {}'\
            .format(ws_covered, len(ws_tokens))
        assert spacy_covered == [n for n in range(len(spacy_tokens))], \
            'WS-SpaCy mapping does not cover all SpaCy tokens: {}; number of tokens: {}' \
            .format(spacy_covered, len(spacy_tokens))

    if remove_stopwords:
        # Filter out stopwords
        nsw_spacy_tokens_lower = list()
        nsw_spacy_lemmas = list()
        for tok_id, tok in enumerate(spacy_tokens_lower):
            if tok not in STOP_WORDS:
                nsw_spacy_tokens_lower.append(tok)
                if get_lemmas:
                    nsw_spacy_lemmas.append(spacy_lemmas[tok_id])
            else:
                if replace_stopwords:
                    nsw_spacy_tokens_lower.append('<STPWRD>')
                    if get_lemmas:
                        nsw_spacy_lemmas.append('<STPWRD>')

        spacy_tokens_lower = nsw_spacy_tokens_lower
        if get_lemmas:
            spacy_lemmas = nsw_spacy_lemmas

    return line_nlp, spacy_tokens_lower, spacy_lemmas, spacy_pos, ws_tokens, ws_tokens_lower, ws_to_spacy_map, \
        spacy_to_ws_map

# TODO / NOTE: Only compatible with word-level attractors, for now
def _get_nmt_label(translation,
                   src,
                   ambiguous_token,
                   ambiguous_token_loc,
                   seed_cluster_id,
                   adv_cluster_id,
                   other_cluster_ids,
                   sense_lemmas_to_cluster,
                   cluster_to_sense_lemmas,
                   sense_tokens_to_cluster,
                   cluster_to_sense_tokens,
                   alignments,
                   tgt_sent):

    """ Helper function for evaluating whether the translations of true and adversarially perturbed source samples
    perform lexical WSD correctly. """

    # Lemmatize the translation for higher coverage of attack successes (preserve punctuation)
    _, spacy_tokens_lower, spacy_lemmas, _, ws_tokens, ws_tokens_lower, ws_to_spacy_map, spacy_to_ws_map = \
        _process_strings(translation,
                         tgt_nlp,
                         get_lemmas=True,
                         get_pos=False,
                         remove_stopwords=False,
                         replace_stopwords=False,
                         get_maps=True)

    # All sequences are Moses-tokenized and can be split at whitespace
    src_tokens = src.strip().split()
    ambiguous_token_loc = ambiguous_token_loc[0]

    # Check that the provided locations of the ambiguous token are correct
    ambiguous_token_in_src = src_tokens[ambiguous_token_loc].lower().strip(punctuation_plus_space)
    assert ambiguous_token.lower().strip(punctuation_plus_space) == \
        ambiguous_token_in_src or ambiguous_token[:-1] in ambiguous_token_in_src, \
        'Ambiguous token \'{:s}\' does not match the true source token \'{:s}\' at the token location'\
        .format(ambiguous_token, ambiguous_token_in_src)

    other_cluster_lemmas = list()
    for cluster_id in other_cluster_ids:
        other_cluster_lemmas += cluster_to_sense_lemmas[cluster_id]
    other_cluster_tokens = list()
    for cluster_id in other_cluster_ids:
        other_cluster_tokens += cluster_to_sense_tokens[cluster_id]

    target_hits = list()
    # If alignments are available, look-up target sense aligned with the ambiguous source term
    alignments = alignments.strip()
    # Convert alignments into a more practical format, first
    line_align_table = dict()
    for word_pair in alignments.strip().split():
        src_id, tgt_id = word_pair.split('-')
        src_id = int(src_id)
        tgt_id = int(tgt_id)
        if not line_align_table.get(src_id, None):
            line_align_table[src_id] = [tgt_id]
        else:
            line_align_table[src_id].append(tgt_id)

    if ambiguous_token_loc in line_align_table.keys():
        homograph_aligned = True
        # Extend alignment window
        line_align_table_entry = sorted(line_align_table[ambiguous_token_loc])
        min_tgt_idx = max(0, line_align_table_entry[0] - ATTN_WINDOW_RANGE)
        max_tgt_idx = min(len(spacy_lemmas) - 1, line_align_table_entry[-1] + ATTN_WINDOW_RANGE)
        # Map from target whitespace to target spacy tokens
        min_tgt_idx = ws_to_spacy_map[min_tgt_idx][0]
        max_tgt_idx = ws_to_spacy_map[max_tgt_idx][0]
        tgt_window = range(min_tgt_idx, max_tgt_idx)
        # Check if aligned translation lemmas include correct / flipped source term translations
        aligned_translation_lemmas = [spacy_lemmas[idx] for idx in tgt_window]
        aligned_tgt_tokens = [spacy_tokens_lower[idx] for idx in tgt_window]

        for atl_id, atl in enumerate(aligned_translation_lemmas):
            # Skip stopwords
            if atl == '<STPWRD>':
                continue
            # Strip leading and trailing punctuation
            atl = atl.strip(punctuation_plus_space)
            # Match lemmas
            if sense_lemmas_to_cluster is not None:
                if sense_lemmas_to_cluster.get(atl, None):
                    target_hits.append(sense_lemmas_to_cluster[atl])
                else:
                    maybe_lemma_matches = sorted(cluster_to_sense_lemmas[seed_cluster_id] +
                                                 cluster_to_sense_lemmas[adv_cluster_id] +
                                                 other_cluster_lemmas, reverse=True, key=lambda x: len(x))

                    for maybe_lemma_match in maybe_lemma_matches:
                        if atl.startswith(maybe_lemma_match) or atl.endswith(maybe_lemma_match) or \
                                atl[:-1].endswith(maybe_lemma_match):
                            target_hits.append(sense_lemmas_to_cluster[maybe_lemma_match])
                            break
            # Match tokens
            if len(target_hits) == 0 and sense_tokens_to_cluster is not None:
                att = aligned_tgt_tokens[atl_id].strip(punctuation_plus_space)
                if sense_tokens_to_cluster.get(att, None):
                    target_hits.append(sense_tokens_to_cluster[att])
                else:
                    maybe_token_matches = \
                        sorted(cluster_to_sense_tokens[seed_cluster_id] +
                               cluster_to_sense_tokens[adv_cluster_id] +
                               other_cluster_tokens, reverse=True, key=lambda x: len(x))

                    for maybe_token_match in maybe_token_matches:
                        if att.startswith(maybe_token_match) or att.endswith(maybe_token_match.lower()) or \
                                att[:-1].endswith(maybe_token_match.lower()):
                            target_hits.append(sense_tokens_to_cluster[maybe_token_match.lower()])
                            break
    else:
        homograph_aligned = False

    # TODO: Can / should this be improved?
    # Check if the attractor term(s) have been translated (assumes translation if alignment was found)
    # NOTE: Currently works ONLY FOR 1-WORD ATTRACTORS (e.g. adjectives)
    is_compound = False

    # If no alignments are available, match the known target sense
    if len(target_hits) == 0:
        for stl_id, stl in enumerate(spacy_lemmas):
            # if stl_id not in range(ambiguous_token_spacy_loc_seed - 3, ambiguous_token_spacy_loc_seed + 4):
            #     continue
            # Skip stopwords
            if stl == '<STPWRD>':
                continue
            # Strip leading and trailing punctuation
            stl = stl.strip(punctuation_plus_space)
            # Match lemmas
            if sense_lemmas_to_cluster is not None:
                if sense_lemmas_to_cluster.get(stl, None):
                    target_hits.append(sense_lemmas_to_cluster[stl])
                else:
                    maybe_lemma_matches = \
                        sorted(cluster_to_sense_lemmas[seed_cluster_id] +
                               cluster_to_sense_lemmas[adv_cluster_id] +
                               other_cluster_lemmas, reverse=True, key=lambda x: len(x))
                    for maybe_lemma_match in maybe_lemma_matches:
                        if stl.startswith(maybe_lemma_match) or stl.endswith(maybe_lemma_match) or \
                                stl[:-1].endswith(maybe_lemma_match):
                            target_hits.append(sense_lemmas_to_cluster[maybe_lemma_match])
                            break
            # Match tokens
            if len(target_hits) == 0 and sense_tokens_to_cluster is not None:
                stt = spacy_tokens_lower[stl_id].strip(punctuation_plus_space)
                if sense_tokens_to_cluster.get(stt, None):
                    target_hits.append(sense_tokens_to_cluster[stt])
                else:
                    maybe_token_matches = \
                        sorted(cluster_to_sense_tokens[seed_cluster_id] +
                               cluster_to_sense_tokens[adv_cluster_id] +
                               other_cluster_tokens, reverse=True, key=lambda x: len(x))
                    for maybe_token_match in maybe_token_matches:
                        if stt.startswith(maybe_token_match) or stt.endswith(maybe_token_match.lower()) or \
                                stt[:-1].endswith(maybe_token_match.lower()):
                            try:
                                target_hits.append(sense_tokens_to_cluster[maybe_token_match.lower()])
                                break
                            except KeyError:
                                pass

    # Source homograph is assumed to be translated if:
    # 1. Homograph is aligned
    # 2. Homograph is not aligned, but len(target_hits) > 0
    if homograph_aligned:
        homograph_translated = True
    else:
        if len(target_hits) > 0:
            homograph_translated = True
        else:
            homograph_translated = False

    # TODO / NOTE: Fixing a corpus bug
    if 'rendezvous' in cluster_to_sense_lemmas[seed_cluster_id]:
        temp = seed_cluster_id
        seed_cluster_id = adv_cluster_id
        adv_cluster_id = temp

    # Flatten target hits
    target_hits = [hit[1] for hit_list in target_hits for hit in hit_list]
    # If target term is ambiguous, assume the translation is correct
    if seed_cluster_id in target_hits:

        return 'not_flipped', target_hits

    elif adv_cluster_id in target_hits:

        # TODO: DEBUGGING
        logging.info('-' * 10)
        logging.info('FLIPPED TO ATTR')
        logging.info(src)
        logging.info(translation)
        logging.info(tgt_sent)
        logging.info('SEED SENSES')
        logging.info(cluster_to_sense_lemmas[seed_cluster_id])
        logging.info('ADVERSARIAL SENSES')
        logging.info(cluster_to_sense_lemmas[adv_cluster_id])

        return 'flipped_to_attr', target_hits

    elif len(set(other_cluster_ids) & set(target_hits)) >= 1:
        return 'flipped_to_other', target_hits

    # i.e. target_hits is empty
    else:
        if homograph_translated:
            return 'maybe_flipped', target_hits

        else:
            return 'deleted_homograph', target_hits

def _build_cluster_lookup(sense_clusters_table):

    """ Post-processes the scraped target sense cluster table by constructing a sense-to-cluster_id lookup table """

    # Initialize empty tables
    logging.info('Constructing the cluster lookup table ...')
    sense_to_cluster_table = dict()

    # Fill tables
    for src_term in sense_clusters_table.keys():
        logging.info('Looking-up the term \'{:s}\''.format(src_term))
        sense_to_cluster_table[src_term] = dict()
        for cluster_id in sense_clusters_table[src_term].keys():
            # Construct cluster-ID lookup table entry
            for tgt_sense in sense_clusters_table[src_term][cluster_id]['[SENSES]']:
                # Lemmatizing single words is not ideal, but is expected to improve attractor recall
                _, _, tgt_lemmas, _, _, _, _, _ = \
                    _process_strings(tgt_sense, tgt_nlp, True, False, False, False, False)
                # Multi-word targets are excluded for simplicity (as a result, some words are dropped)
                if len(tgt_lemmas) < 1:
                    continue
                tgt_lemma = tgt_lemmas[0]
                if len(tgt_lemma) > 0:
                    if not sense_to_cluster_table[src_term].get(tgt_lemma, None):
                        sense_to_cluster_table[src_term][tgt_lemma] = [(tgt_sense, cluster_id, True)]
                    else:
                        sense_to_cluster_table[src_term][tgt_lemma].append((tgt_sense, cluster_id, True))

            # Check for blacklisted and ambiguous senses
            senses_to_ignore = list()
            senses_to_ignore += sense_clusters_table[src_term][cluster_id].get('[BLACKLISTED SENSES]', [])
            senses_to_ignore += sense_clusters_table[src_term][cluster_id].get('[AMBIGUOUS SENSES]', [])
            for tgt_sense in senses_to_ignore:
                # Lemmatizing single words is not ideal, but is expected to improve attractor recall
                _, _, tgt_lemmas, _, _, _, _, _ = \
                    _process_strings(tgt_sense, tgt_nlp, True, False, False, False, False)
                # Multi-word targets are excluded for simplicity (as a result, some words are dropped)
                if len(tgt_lemmas) < 1:
                    continue
                tgt_lemma = tgt_lemmas[0]
                if len(tgt_lemma) > 0:
                    if not sense_to_cluster_table[src_term].get(tgt_lemma, None):
                        sense_to_cluster_table[src_term][tgt_lemma] = [(tgt_sense, cluster_id, False)]
                    else:
                        sense_to_cluster_table[src_term][tgt_lemma].append((tgt_sense, cluster_id, False))
    return sense_to_cluster_table

def evaluate_attack_success(wsd_samples_path,
                            sources_path,
                            translations_path,
                            alignments_path,
                            attractors_path,
                            sense_clusters_path,
                            output_tables_dir):

    """ Detects successful attacks and computes correlation between attack success and various metrics. """

    def _score_and_filter(combined_sample, ambiguous_term, attractors_entry):
        """ Helper function for filtering a single adversarial sample. """

        # Combined sample contents:

        # [translations[line_id],
        # alignments[line_id],
        # src_to_sample[src_line.strip()][0],
        # src_to_sample[src_line.strip()][1]]

        sense_lemmas_to_cluster = sense_lemmas_to_cluster_table.get(ambiguous_term, None)
        sense_tokens_to_cluster = sense_tokens_to_cluster_table.get(ambiguous_term, None)
        cluster_to_sense_lemmas = cluster_to_sense_lemmas_table.get(ambiguous_term, None)
        cluster_to_sense_tokens = cluster_to_sense_tokens_table.get(ambiguous_term, None)

        # Unpack
        translation = combined_sample[0]
        line_alignments = combined_sample[1]
        challenge_entry = combined_sample[2]

        src_sent = challenge_entry[1]
        tgt_sent = challenge_entry[2]
        ambiguous_token_loc_seed = challenge_entry[7]
        ambiguous_token_ws_loc_seed = challenge_entry[8]  # list
        seed_cluster_num = challenge_entry[12]
        adv_cluster_num = challenge_entry[13]

        other_cluster_ids = list(attractors_entry.keys())
        other_cluster_ids.pop(other_cluster_ids.index(seed_cluster_num))
        try:
            other_cluster_ids.pop(other_cluster_ids.index(adv_cluster_num))
        except ValueError:
            pass

        # Ignore samples containing multiple instances of the attractor term
        src_tokens = [tok.strip(punctuation_plus_space) for tok in src_sent.split()]
        src_tokens = [tok for tok in src_tokens if len(tok) > 0]
        # Ignore short sentences
        if len(src_tokens) < 10:
            return None

        # Get NMT labels
        true_nmt_label, true_translation_sense_clusters = \
            _get_nmt_label(translation,
                           src_sent,
                           ambiguous_term,
                           ambiguous_token_ws_loc_seed,
                           seed_cluster_num,
                           adv_cluster_num,
                           other_cluster_ids,
                           sense_lemmas_to_cluster,
                           cluster_to_sense_lemmas,
                           sense_tokens_to_cluster,
                           cluster_to_sense_tokens,
                           line_alignments,
                           tgt_sent)

        # Sort true samples into appropriate output tables, based on filtering outcome
        seed_table_to_expand = None
        if true_nmt_label == 'not_flipped':
            if not unique_samples.get((src_sent, ambiguous_term, ambiguous_token_loc_seed), None):
                stat_dict['num_true_samples_good_translations'] += 1
                unique_samples[(src_sent, ambiguous_term, ambiguous_token_loc_seed)] = True
                seed_table_to_expand = true_samples_good_translations
        elif true_nmt_label == 'flipped_to_attr':
            if not unique_samples.get((src_sent, ambiguous_term, ambiguous_token_loc_seed), None):
                stat_dict['num_true_samples_bad_translations'] += 1
                unique_samples[(src_sent, ambiguous_term, ambiguous_token_loc_seed)] = True
                seed_table_to_expand = true_samples_bad_translations
        elif true_nmt_label == 'flipped_to_other':
            if not unique_samples.get((src_sent, ambiguous_term, ambiguous_token_loc_seed), None):
                stat_dict['num_true_samples_bad_translations'] += 1
                unique_samples[(src_sent, ambiguous_term, ambiguous_token_loc_seed)] = True
                seed_table_to_expand = true_samples_bad_translations
        elif true_nmt_label == 'deleted_homograph':
            if not unique_samples.get((src_sent, ambiguous_term, ambiguous_token_loc_seed), None):
                stat_dict['num_true_samples_deleted_homograph'] += 1
                unique_samples[(src_sent, ambiguous_term, ambiguous_token_loc_seed)] = True
                seed_table_to_expand = true_samples_deleted_homograph
        else:
            if not unique_samples.get((src_sent, ambiguous_term, ambiguous_token_loc_seed), None):
                stat_dict['num_true_samples_maybe_bad_translations'] += 1
                unique_samples[(src_sent, ambiguous_term, ambiguous_token_loc_seed)] = True
                seed_table_to_expand = true_samples_maybe_bad_translations

        # Collect seed translations
        if seed_table_to_expand is not None:
            if not seed_table_to_expand.get(ambiguous_term, None):
                seed_table_to_expand[ambiguous_term] = dict()
            if not seed_table_to_expand[ambiguous_term].get(seed_cluster_num, None):
                seed_table_to_expand[ambiguous_term][seed_cluster_num] = dict()
            if not seed_table_to_expand[ambiguous_term][seed_cluster_num].get(adv_cluster_num, None):
                seed_table_to_expand[ambiguous_term][seed_cluster_num][adv_cluster_num] = dict()
            if not seed_table_to_expand[ambiguous_term][seed_cluster_num][adv_cluster_num].get(src_sent, None):
                seed_table_to_expand[ambiguous_term][seed_cluster_num][adv_cluster_num][src_sent] = list()
            seed_table_to_expand[ambiguous_term][seed_cluster_num][adv_cluster_num][src_sent]\
                .append([translation, tgt_sent, true_translation_sense_clusters, ambiguous_token_loc_seed])

    def _show_stats():
        """ Helper for reporting on the generation process. """

        def _pc(enum, denom):
            """ Helper function for computing percentage of the evaluated sample type """
            return (enum / denom) * 100 if denom > 0 else 0

        logging.info('-' * 20)

        num_all_seed_samples = \
            stat_dict['num_true_samples_good_translations'] + \
            stat_dict['num_true_samples_bad_translations'] + \
            stat_dict['num_true_samples_deleted_homograph'] + \
            stat_dict['num_true_samples_maybe_bad_translations']

        # ==============================================================================================================
        logging.info('Evaluated {:d} seed samples'.format(num_all_seed_samples))
        logging.info('{:d} ({:.4f}%) TRUE source sentences have been translated CORRECTLY'
                     .format(stat_dict['num_true_samples_good_translations'],
                             _pc(stat_dict['num_true_samples_good_translations'], num_all_seed_samples)))

        logging.info('{:d} ({:.4f}%) TRUE source sentences have been translated INCORRECTLY'
                     .format(stat_dict['num_true_samples_bad_translations'],
                             _pc(stat_dict['num_true_samples_bad_translations'], num_all_seed_samples)))

        logging.info('{:d} ({:.4f}%) TRUE source sentences have been translated with a DELETED HOMOGRAPH'
                     .format(stat_dict['num_true_samples_deleted_homograph'],
                             _pc(stat_dict['num_true_samples_deleted_homograph'], num_all_seed_samples)))

        logging.info('{:d} ({:.4f}%) TRUE source sentences have been translated MAYBE INCORRECTLY'
                     .format(stat_dict['num_true_samples_maybe_bad_translations'],
                             _pc(stat_dict['num_true_samples_maybe_bad_translations'], num_all_seed_samples)))

        logging.info('{:d} ({:.4f}%) ALL TRUE source sentences have been translated WRONG'
                     .format(stat_dict['num_true_samples_bad_translations'] +
                             stat_dict['num_true_samples_deleted_homograph'] +
                             stat_dict['num_true_samples_maybe_bad_translations'],
                             100 - _pc(stat_dict['num_true_samples_good_translations'], num_all_seed_samples)))

    # Read-in adversarial samples
    logging.info('Reading-in challenge samples table ...')
    with open(wsd_samples_path, 'r', encoding='utf8') as wsp:
        wsd_samples_table = json.load(wsp)

    # Read-in attractor table
    logging.info('Reading-in attractor table ...')
    with open(attractors_path, 'r', encoding='utf8') as ap:
        attractors_table = json.load(ap)

    # Load sense to cluster mappings
    logging.info('Reading-in sense clusters ...')
    with open(sense_clusters_path, 'r', encoding='utf8') as scp:
        sense_clusters_table = json.load(scp)

    # Add blacklisted, ambiguous terms to the set of acceptable translations
    for term in sense_clusters_table.keys():
        for cluster in sense_clusters_table[term].keys():
            if '[AMBIGUOUS SENSES]' in sense_clusters_table[term][cluster].keys():
                sense_clusters_table[term][cluster]['[SENSES]'] += \
                    sense_clusters_table[term][cluster]['[AMBIGUOUS SENSES]']
                sense_clusters_table[term][cluster].pop('[AMBIGUOUS SENSES]')
            if '[BLACKLISTED SENSES]' in sense_clusters_table[term][cluster].keys():
                sense_clusters_table[term][cluster]['[SENSES]'] += \
                    sense_clusters_table[term][cluster]['[BLACKLISTED SENSES]']
                sense_clusters_table[term][cluster].pop('[BLACKLISTED SENSES]')
            sense_clusters_table[term][cluster]['[SENSES]'] = \
                list(set(sense_clusters_table[term][cluster]['[SENSES]']))
    sense_to_cluster_table = _build_cluster_lookup(sense_clusters_table)

    # Post-process sense-to-cluster table
    sense_lemmas_to_cluster_table = sense_to_cluster_table
    sense_tokens_to_cluster_table = dict()
    for src_term in sense_lemmas_to_cluster_table.keys():
        sense_tokens_to_cluster_table[src_term] = dict()
        for cluster_tuple_list in sense_lemmas_to_cluster_table[src_term].values():
            for cluster_tuple in cluster_tuple_list:
                sense_tokens_to_cluster_table[src_term][cluster_tuple[0].lower()] = [cluster_tuple]

    cluster_to_sense_lemmas_table = dict()
    # Derive a cluster-to-senses table, used for compound analysis
    for src_term in sense_lemmas_to_cluster_table.keys():
        cluster_to_sense_lemmas_table[src_term] = dict()
        for sense, clusters in sense_lemmas_to_cluster_table[src_term].items():
            for cls_tpl in clusters:
                if not cluster_to_sense_lemmas_table[src_term].get(cls_tpl[1], None):
                    cluster_to_sense_lemmas_table[src_term][cls_tpl[1]] = [sense]
                else:
                    if sense not in cluster_to_sense_lemmas_table[src_term][cls_tpl[1]]:
                        cluster_to_sense_lemmas_table[src_term][cls_tpl[1]].append(sense)

    cluster_to_sense_tokens_table = dict()
    # Derive a cluster-to-senses table, used for compound analysis
    for src_term in sense_tokens_to_cluster_table.keys():
        cluster_to_sense_tokens_table[src_term] = dict()
        for sense, clusters in sense_lemmas_to_cluster_table[src_term].items():
            for cls_tpl in clusters:
                if not cluster_to_sense_tokens_table[src_term].get(cls_tpl[1], None):
                    cluster_to_sense_tokens_table[src_term][cls_tpl[1]] = [cls_tpl[0]]
                else:
                    if sense not in cluster_to_sense_tokens_table[src_term][cls_tpl[1]]:
                        cluster_to_sense_tokens_table[src_term][cls_tpl[1]].append(cls_tpl[0])

    # Restructure NMT translations for easier access
    logging.info('-' * 10)
    logging.info('Hashing challenge set samples ...')
    with open(sources_path, 'r', encoding='utf8') as sp:
        sources = [line for line_id, line in enumerate(sp)]
    with open(translations_path, 'r', encoding='utf8') as tp:
        translations = [line for line_id, line in enumerate(tp)]
    with open(alignments_path, 'r', encoding='utf8') as ap:
        alignments = [line for line_id, line in enumerate(ap)]
    logging.info('-' * 10)

    # Create a adversarial-to-original map
    src_to_sample = dict()
    for term in wsd_samples_table.keys():
        for true_cluster_id in wsd_samples_table[term].keys():
            for adv_cluster_id in wsd_samples_table[term][true_cluster_id].keys():
                for smp in wsd_samples_table[term][true_cluster_id][adv_cluster_id]:
                    src = smp[1].strip()
                    src_to_sample[src] = (smp, term)

    # Combine sources, samples, translations, and alignments
    logging.info('Mapping translations...')
    src_to_info = {src_line: [translations[line_id],
                              alignments[line_id],
                              src_to_sample[src_line.strip()][0],
                              src_to_sample[src_line.strip()][1]] for line_id, src_line
                   in enumerate(sources)}

    # For stats
    unique_samples = dict()

    # Seed translations
    true_samples_good_translations = dict()
    true_samples_bad_translations = dict()
    true_samples_deleted_homograph = dict()
    true_samples_maybe_bad_translations = dict()

    # Initialize variables for reporting
    stat_dict = {
        'num_true_samples_good_translations': 0,
        'num_true_samples_bad_translations': 0,
        'num_true_samples_deleted_homograph': 0,
        'num_true_samples_maybe_bad_translations': 0,
    }

    sent_id = 0
    for sent_id, sent in enumerate(src_to_info.keys()):

        sample = src_to_info[sent]
        term = sample[3]

        # TODO: DEBUGGING
        # if term not in ['anchor', 'clip']:
        #     continue

        # Apply adversarial filtering and compute LM-based fluency / acceptability scores
        _score_and_filter(sample, term, attractors_table[term])

        # Occasionally report statistics
        if sent_id > 0 and sent_id % 1000 == 0:
            logging.info('\nLooked up {:d} sentences; reporting intermediate statistics:'.format(sent_id + 1))
            _show_stats()

        # TODO: DEBUGGING
        # if term_id == 2:
        #     break

    # Final report
    logging.info('\nLooked up {:d} sentences; reporting FINAL statistics:'.format(sent_id + 1))
    _show_stats()

    # Construct output paths
    table_list = [
        true_samples_good_translations,
        true_samples_bad_translations,
        true_samples_deleted_homograph,
        true_samples_maybe_bad_translations,
    ]

    attack_success_path = os.path.join(output_tables_dir, 'attack_success')

    if not os.path.isdir(attack_success_path):
        os.mkdir(attack_success_path)

    path_list = [
        os.path.join(attack_success_path, 'wsd_samples_good_translations.{:s}'.format(src_lang)),
        os.path.join(attack_success_path, 'wsd_samples_bad_translations.{:s}'.format(src_lang)),
        os.path.join(attack_success_path, 'wsd_samples_deleted_homograph.{:s}'.format(src_lang)),
        os.path.join(attack_success_path, 'wsd_samples_maybe_bad_translations.{:s}'.format(src_lang)),
    ]

    # Save
    for table, path in zip(table_list, path_list):
        with open(path + '.json', 'w', encoding='utf8') as json_file:
            json.dump(table, json_file, indent=3, sort_keys=True, ensure_ascii=False)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--wsd_samples_path', type=str, required=True,
                        help='path to the JSON file containing selected challenge samples')
    parser.add_argument('--sources_path', type=str, default=None,
                        help='path to the text file containing the true source sentences')
    parser.add_argument('--translations_path', type=str, default=None,
                        help='path to the translations produced by the filter NMT model for the true source sentences')
    parser.add_argument('--alignments_path', type=str, default=None,
                        help='path to file containing the fastalign alignments for the translations of the original '
                             'challenge sentences')
    parser.add_argument('--sense_clusters_path', type=str, default=None,
                        help='path to the JSON file containing scraped BabelNet sense clusters')
    parser.add_argument('--attractors_path', type=str, required=True,
                        help='path to the JSON file containing the extracted attractor terms')
    parser.add_argument('--output_tables_dir', type=str, required=True,
                        help='path to the destination of the filtered adversarial samples tables')
    parser.add_argument('--lang_pair', type=str, default=None,
                        help='language pair of the bitext; expected format is src-tgt')
    args = parser.parse_args()

    # Logging to file
    base_dir = '/'.join(args.wsd_samples_path.split('/')[:-1])
    file_name = args.wsd_samples_path.split('/')[-1]
    file_name = '.'.join(file_name.split('.')[:-1])
    file_name = file_name if len(file_name) > 0 else 'log'
    log_dir = '{:s}/logs/'.format(base_dir)
    if not os.path.exists(log_dir):
        os.mkdir(log_dir)
    log_path = '{:s}{:s}.log'.format(log_dir, file_name)
    logging.basicConfig(filename=log_path, filemode='w', level=logging.INFO, format='%(levelname)s: %(message)s')
    # Logging to console
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    logging.getLogger('').addHandler(console)

    # Instantiate processing pipeline
    src_lang, tgt_lang = args.lang_pair.strip().split('-')
    spacy_map = {'en': 'en_core_web_sm', 'de': 'de_core_news_sm'}
    try:
        src_nlp = spacy.load(spacy_map[src_lang], disable=['parser', 'ner', 'textcat'])
        tgt_nlp = spacy.load(spacy_map[tgt_lang], disable=['parser', 'ner', 'textcat'])
    except KeyError:
        logging.info('SpaCy does not support the language {:s} or {:s}. Exiting.'.format(src_lang, tgt_lang))
        sys.exit(0)
    pct_stripper = str.maketrans(string.punctuation, ' ' * len(string.punctuation))
    punctuation_plus_space = string.punctuation + ' ' + '\t' + '\n'
    # Import stopword list
    if src_lang == 'en':
        STOP_WORDS = stopwords.words('english')
    else:
        STOP_WORDS = []

    evaluate_attack_success(args.wsd_samples_path,
                            args.sources_path,
                            args.translations_path,
                            args.alignments_path,
                            args.attractors_path,
                            args.sense_clusters_path,
                            args.output_tables_dir)


# This file has all functions: AddNdvi,removenn,intense,predata

# function to add NDVI band to image
def clip(image):
    import ee
    ee.Initialize()
    bound = ee.Geometry.Rectangle([89.985917,25.019917,90.799167,23.498889])
    return image.clip(bound)

def AddNdvi(image):
    red=image.select('sur_refl_b01')
    nir=image.select('sur_refl_b02')
    ndvi = (nir.subtract(red)).divide(nir.add(red)).rename('ndvi')
    return image.addBands(ndvi)

# turn the Pandas timestamp to day of year
def dayyear(list):
    dayn=[0]*len(list)
    for d in range(len(list)):
        dayn[d] = list[d].to_pydatetime().timetuple().tm_yday
    return dayn

# remove null and negative values from NDVI
def removenn(list):
    x=[0]*len(list)
    for i in range(len(list)):
        value=list[i]
        if value == None or value < 0:
            if i!= (len(list)-1):
                y=list[i+1]
                if y==None or y<0:
                    x[i]=list[i-1]
                    list[i]=list[i-1]
                else:
                    x[i]=(list[i-1]+list[i+1])/2
            else:
                x[i]=list[i-1]
        else:
            x[i]=list[i]
    return x

# detect the intensity and local maximum NDVI (highest value existing within 3 month)
def intense(list):
    intensity=0
    maxv=[]
    maxindex = []
    slist=[0]*2
    minv=[0]*2
    minindex=[0]*2
    n=len(list)
    for j in range(5,n-5):
        newlist=[list[j-5],list[j-4],list[j-3],list[j-2],list[j-1],list[j],list[j+1],list[j+2],list[j+3],list[j+4],list[j+5]]
        if list[j]==max(newlist) and list[j]>0.5:
            intensity=intensity+1
            print('a')
            maxv.append(list[j])
            maxindex.append(j)
    if intensity>3:
        print("There is something wrong in intensity")
    elif intensity==2:
        slist[0]=list[maxindex[0]:maxindex[1]+1]
        minv[0]=min(slist[0])
        minindex[0]=maxindex[0]+slist[0].index(min(slist[0])) # the minindex = first max + min location within small list
    elif intensity==3:
        slist[0] = list[maxindex[0]:maxindex[1] + 1]
        slist[1] = list[maxindex[1]:maxindex[2] + 1]
        minv[0] = min(slist[0])
        minv[1] = min(slist[1])
        minindex[0] = maxindex[0] + slist[0].index(min(slist[0]))
        minindex[1] = maxindex[1] + slist[1].index(min(slist[1]))
    return intensity,maxv,maxindex,minv,minindex

# return the ndvi data into the y as sigmoid function
def predata(list):
    mindata=min(list)
    maxdata=max(list)
    newlist=[0]*len(list)
    for i in range(len(list)):
        if list[i]==min(list):
            list[i]=list[i]+0.001
        elif list[i]==max(list):
            list[i]=list[i]-0.001
        newlist[i]=(list[i]-mindata)/(maxdata-mindata)
    return newlist

# sigmoid function used for the curve-fitting
def sigmoid(x, a, b):
    import numpy as np
    y = 1/(1+np.exp(a+b*x))
    return y

# curve-fitting
def curvefit(list1,list2):
    from scipy.optimize import curve_fit
    if list2[-1]>0.5:
        popt, pcov = curve_fit(sigmoid, list1, list2,p0=(20,-0.1))
    else:
        popt, pcov = curve_fit(sigmoid, list1, list2, p0=(-20, 0.1))
    return popt,pcov

# generate the ndvi date for visualization/validation
def validate(list1,x1,x2,list2):
    import numpy as np
    a=list2[0]
    b=list2[1]
    y=[0]*len(list1)
    for i in range(len(list1)):
        y[i]=x1+(x2-x1)*(1/(1+np.exp(a+b*list1[i])))
    return y

# calculate the change rate of curvature
# and then find the four days (greenup date GD, maturity date MD, senescence date SD, dormancy DD)
def curverate(list1,x1,x2,list2):
    import numpy as np
    a=list2[0]
    b=list2[1]
    c=x2-x1
    allday=range(list1[0],list1[-1]+1)
    ccr=[0]*len(allday)
    y=[0]*len(allday)
    greenD=0
    matureD=0
    senesD=0
    dormanD=0
    for i in range(len(allday)):
        z=np.exp(a+b*allday[i])
        y[i]=(c/(1+z))+x1
        s1=3*z*(1-z)*((1+z)**3)*(2*((1+z)**3)+(b**2)*(c**2)*z)
        s2=(((1+z)**4)+((b*c*z)**2))**2.5
        s3=((1+z)**2)*(1+2*z-5*(z**2))
        s4=(((1+z)**4)+((b*c*z)**2))**1.5
        ccr[i]=((b**3)*c*z*((s1/s2)-(s3/s4)))
    if b<0:
        top2index=sorted(np.argsort(ccr)[-2:])
    else:
        top2index = sorted(np.argsort(ccr)[:2])
    detectd1=[allday[i] for i in top2index][0]
    detectd2=[allday[i] for i in top2index][1]
    return y,ccr,detectd1,detectd2,allday


import numpy
import matplotlib.pyplot as plt
import pika
import os
import sys
import json
import time

class VisualizationOutput:
    def __init__(self, json):
        self.output_user_pos_x = float(json['output_user_pos_x_to_visualization'])
        self.output_user_pos_y = float(json['output_user_pos_y_to_visualization'])
        self.output_user_pos_z = float(json['output_user_pos_z_to_visualization'])
        self.output_op_pos_x = float(json['output_op_pos_x_to_visualization'])
        self.output_op_pos_y = float(json['output_op_pos_y_to_visualization'])
        self.output_op_pos_z = float(json['output_op_pos_z_to_visualization'])
        self.output_op_haptics_x = float(json['HapticsX'])
        self.output_op_haptics_y = float(json['HapticsY'])
        self.output_op_haptics_z = float(json['HapticsZ'])
        self.output_op_step = int(json['Step'])

class Visualizer:
    def __init__(self):
        self.count = 0
        """graph arrays of values, to be on graph"""
        self.op_x_array = numpy.empty([0])
        self.op_y_array = numpy.empty([0])
        self.op_z_array = numpy.empty([0])

        self.user_x_array = numpy.empty([0])
        self.user_y_array = numpy.empty([0])
        self.user_z_array = numpy.empty([0])

        self.haptics_x_array = numpy.empty([0])
        self.haptics_y_array = numpy.empty([0])
        self.haptics_z_array = numpy.empty([0])
        self.haptics_step_array = numpy.empty([0])

        self.lasttenxhaptics = 0.0
        self.lasttenyhaptics = 0.0
        self.lasttenzhaptics = 0.0

        """interactive on"""
        plt.ion()
        self.fig = plt.figure()
        self.ax = self.fig.add_subplot(projection='3d')
        self.ax.legend()
        self.ax.set_xlabel('X')
        self.ax.set_ylabel('Y')
        self.ax.set_zlabel('Z')

        self.hapticsax = plt.figure().add_subplot(1,1,1)
        self.hapticsax.legend()
        self.hapticsax.set_xlabel('X')
        self.hapticsax.set_ylabel('Y')

    def callback(self, ch, method, properties, body):

        print(" [x] Received %r" % body)

        message = json.loads(body)

        visualizationOutput = VisualizationOutput(message)

        visualizationOutput.output_user_pos_x += self.count

        #time.sleep(2)

        """make graph increase"""
        self.op_x_array = numpy.append(self.op_x_array, visualizationOutput.output_op_pos_x)
        self.op_y_array = numpy.append(self.op_y_array, visualizationOutput.output_op_pos_y)
        self.op_z_array = numpy.append(self.op_z_array, visualizationOutput.output_op_pos_z)

        self.user_x_array = numpy.append(self.user_x_array, visualizationOutput.output_user_pos_x)
        self.user_y_array = numpy.append(self.user_y_array, visualizationOutput.output_user_pos_y)
        self.user_z_array = numpy.append(self.user_z_array, visualizationOutput.output_user_pos_z)

        self.haptics_x_array = numpy.append(self.haptics_x_array, visualizationOutput.output_op_haptics_x)
        self.haptics_y_array = numpy.append(self.haptics_y_array, visualizationOutput.output_op_haptics_y)
        self.haptics_z_array = numpy.append(self.haptics_z_array, visualizationOutput.output_op_haptics_z)
        self.haptics_step_array = numpy.append(self.haptics_step_array, visualizationOutput.output_op_step)

        if(self.haptics_step_array[-1] % 10 == 0):
            self.lasttenxhaptics = self.calculateAbsOfArray(self.haptics_x_array[-10:])
            self.lasttenyhaptics = self.calculateAbsOfArray(self.haptics_y_array[-10:])
            self.lasttenzhaptics = self.calculateAbsOfArray(self.haptics_z_array[-10:])

        self.ax.annotate('Accumulated haptic \n feedback for last 10 readings:', xy=(-0.10, 0.01), xytext=(0, 10), xycoords=('axes fraction', 'figure fraction'), textcoords='offset points', ha='center', va='bottom')
        self.ax.annotate(f'x: {self.lasttenxhaptics}', xy=(0.5, 0.035), xytext=(0, 10), xycoords=('axes fraction', 'figure fraction'), textcoords='offset points', ha='center', va='bottom')
        self.ax.annotate(f'y: {self.lasttenyhaptics}', xy=(0.5, 0.01), xytext=(0, 10), xycoords=('axes fraction', 'figure fraction'), textcoords='offset points', ha='center', va='bottom')
        self.ax.annotate(f'z: {self.lasttenzhaptics}', xy=(0.5, -0.015), xytext=(0, 10), xycoords=('axes fraction', 'figure fraction'), textcoords='offset points', ha='center', va='bottom')

        self.ax.plot(self.op_x_array, self.op_y_array, zs=self.op_z_array, zdir='z', color="black", label='Optimal Path')
        self.ax.plot(self.user_x_array, self.user_y_array, zs=self.user_z_array, zdir='z', color="green", label='User Path')
        self.ax.legend()

        self.hapticsax.plot(self.haptics_step_array, self.haptics_x_array, label='x-axis')
        self.hapticsax.plot(self.haptics_step_array, self.haptics_y_array, label='y-axis')
        self.hapticsax.plot(self.haptics_step_array, self.haptics_z_array, label='z-axis')
        self.hapticsax.legend()

        plt.draw()
        plt.pause(0.5)

        self.ax.cla()

        self.hapticsax.cla()

        self.ax.set_xlabel('X')
        self.ax.set_ylabel('Y')
        self.ax.set_zlabel('Z')

        ch.basic_ack(delivery_tag=method.delivery_tag)

    def calculateAbsOfArray(self, arr):
        total = 0.0
        for value in arr:
            total += abs(value)

        return total

def main():
    visualization_queue = "visualization"

    connectionParameters = pika.ConnectionParameters(host = 'localhost', port = 5672)

    connection = pika.BlockingConnection(connectionParameters)

    visuazer = Visualizer()

    channel = connection.channel()

    channel.basic_qos(prefetch_count = 1)

    channel.basic_consume(queue=visualization_queue,
                        auto_ack=False,
                        on_message_callback=visuazer.callback)

    print(' [*] Waiting for messages. To exit press CTRL+C')
    channel.start_consuming()

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        print('Interrupted')
        try:
            sys.exit(0)
        except SystemExit:
            os._exit(0)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import os.path
import pynotify
import urllib
import xml.dom.minidom
#import httplib

class Forismatic:
	"""Return random quote form www.forismatic.com"""
	def __init__(self, conf,path = None):
		"""init some variables"""
		self.conf = conf
		if path == None:
			self.path = '/usr/share/ForismaticApplet'
		else:
			self.path = path

	def get_quote(self):
		"""Read last quote from gconf (or file)"""
		#saved_quote_path=os.path.join(cur_dir, 'forismatic') # if quote write to file
		#print logo_path
		#0) read file with saved quote
		"""with codecs.open(saved_quote_path,
						encoding='utf-8',mode='r') as file_r:
			quote = file_r.readlines()
		file_r.closed
		quote=u"".join(quote)
		"""
		#read from GConf
		(self.QuoteText,self.quoteAuthor,self.QuoteLink)=self.conf.get_Quote()
		#print "%s  \n   %s :: %s"%(self.QuoteText,self.quoteAuthor,self.QuoteLink)
		if self.quoteAuthor == "":
			quote=self.QuoteText
		else:
			quote=self.QuoteText+"\n"+self.quoteAuthor
		#print quote
		return quote

	def show_notify(self,body):
		"""show quote using Notification"""
		(self.TimePeriod,self.Theme, self.Lang) = self.conf.get_settings()
		cur_dir=os.path.abspath(self.path)
		self.theme_path = os.path.join(self.path,'Themes',self.Theme)
		logo_path=os.path.join(self.theme_path,'forismatic_logo48x48.png')
		#print logo_path
		title='Forismatic'
		icon='file://'+logo_path
		notif = pynotify.Notification (title, body, icon);
		notif.show ()

	def save_quote(self):
		"""Saved new quote to gconf"""
		#1)get quote form forismatic
		#exampl of quote:
		"""
		<forismatic>
			<quote>
				<quoteText>
					Для любви не существует вчера, любовь не думает \
					о завтра.
					Она жадно тянется к нынешнему дню, но этот день \
					нужен ей весь, неограниченный, не омраченный.
				</quoteText>
				<quoteAuthor>
					Генрих Гейне
				</quoteAuthor>
				<senderName>
				</senderName>
				<senderLink>
					http://ru.forismatic.com/8d9098bc35/
				</senderLink>
				<quoteLink>
					http://ru.forismatic.com/13d15ebb9a/
				</quoteLink>
			</quote>
		</forismatic>
		"""
		#TODO: add check of bad response
		self.lang = u"".join(self.quote_lang())
		params = urllib.urlencode({ 'format': 'xml','key': '',
			'lang': self.lang,'method': 'getQuote' })
		#new version
		"""
			headers = {"Content-type": "application/x-www-form-urlencoded",
				"Accept": "*/*"}
			conn = httplib.HTTPConnection('api.forismatic.com')
			conn.request("POST", "/api/1.0/", params, headers)
			response = conn.getresponse()
			print response.status, response.reason
			quote_xml = response.read()
			conn.close()
		"""
		#old version:
		f =urllib.urlopen("http://api.forismatic.com/api/1.0/",params)
		quote_xml = f.read()

		#2)parseQuote:
		quoteText = "can not get quote"
		quoteAuthor = ""
		quoteLink = "error"

		quote=xml.dom.minidom.parseString(quote_xml)
		#text
		text = quote.getElementsByTagName('quoteText')
		for node in text[0].childNodes:
			if node.nodeType == node.TEXT_NODE:
				quoteText=node.data
		#author
		text = quote.getElementsByTagName('quoteAuthor')
		for node in text[0].childNodes:
			if node.nodeType == node.TEXT_NODE:
				quoteAuthor =node.data
				quoteAuthor = "--"+ quoteAuthor
				quoteAuthor = quoteAuthor.rjust(30," ")
		#link
		text = quote.getElementsByTagName('quoteLink')
		for node in text[0].childNodes:
			if node.nodeType == node.TEXT_NODE:
				quoteLink =node.data
		#write to file
		"""with codecs.open(saved_quote_path,
						encoding='utf-8',mode='w') as file_w:
			file_w.write(quoteText)
			file_w.write("\n"+quoteAuthor)
		file_w.closed"""
		#write to GConf
		self.conf.set_Quote(quoteText, quoteAuthor, quoteLink)
		#print "%s \n     %s \n  %s "%(quoteText, quoteAuthor, quoteLink)
		#print "end"

	def quote_lang(self):
		"""Return current language for quote"""
		self.Language=self.conf.get_settings()[2]
		if self.Language == 'English' or self.Language == 'english' \
			or self.Language == 'en':
			return 'en'
		elif self.Language == 'Russian' or self.Language == 'russian' \
			or self.Language == 'ru':
			return 'ru'
		else:
			return None

if __name__ == '__main__':
	import Preference
	conf = Preference.Config()
	path = os.getcwd()

	quote=Forismatic(conf,path)
	quote.save_qote()


from __future__ import print_function

import numpy as np
import tensorflow as tf
from collections import defaultdict
from IPython import embed
import matplotlib.pyplot as plt

DATA_LOC = './assistments.txt'
MAX_LENGTH = 100
LR = 0.0005
HIDDEN_SIZE = 200
BATCH_SIZE = 32
MAX_EPOCHS = 10
DROPOUT = 0.3
TRAIN_SPLIT = 0.7
EMBEDDING_SIZE = 25

def read_assistments_data(loc):
    with open(loc) as f:
        data = [[int(y) for y in x.strip().split()] for x in f.readlines()]

    topic_seqs = defaultdict(list)
    answer_seqs = defaultdict(list)
    topics = set()

    for d in data:
        student_id = d[0]
        if(len(topic_seqs[student_id])) >= MAX_LENGTH: continue
        topic_seqs[student_id].append(d[1])
        answer_seqs[student_id].append(d[2])
        topics.add(d[1])

    #keys = topic_seqs.keys()
    #np.random.shuffle(keys)
    #return [topic_seqs[k] for k in keys], [answer_seqs[k] for k in keys], max(topics) + 1
    #topics may not be consecutively numbered but assume they are
    order = list(topic_seqs.keys())

    return [topic_seqs[x] for x in order], [answer_seqs[x] for x in order], max(topics) + 1

# Takes jagged arrays of topic and answer sequences
# Pads them into square arrays; returns that and a mask
def load_data(topic_seqs, answer_seqs, num_topics):
    assert(len(topic_seqs) == len(answer_seqs))
    N = len(topic_seqs)

    topics_padded = []
    answers_padded = []
    masks = []
    embeddings = np.zeros((N, MAX_LENGTH + 1, num_topics*2))
    seq_lens = []

    for i in range(N):
        assert(len(topic_seqs[i]) == len(answer_seqs[i]))
        seq_len = len(topic_seqs[i])
        seq_lens.append(seq_len)
        padding = [0] * (MAX_LENGTH - seq_len)
        topics_padded.append(topic_seqs[i] + padding)
        answers_padded.append(answer_seqs[i] + padding)
        masks.append([1] * seq_len + padding)
        #TODO: we can actually modify the code so that we predict the
        #probabilities of correct answers *after* the sequence ends as well.
        #This requires modifying some shapes as well as doing one more step
        #in this loop
        for j in range(seq_len):
            # Make the one-hot representation; for each sequence and time step,
            # 2*n_topics length vector, one-hot for topic/answer index
            one_hot_index = topic_seqs[i][j] * 2 + answer_seqs[i][j]
            embeddings[i][j + 1][one_hot_index] = 1

    #padded topics and answers not actually used, could remove?
    topics = topics_padded
    answers = answers_padded
    #return embeddings, seq_lens, masks, answers, topics
    return seq_lens, masks, answers, topics

class DKTModel(object):

    def add_placeholders(self):
        self.topics_placeholder = tf.placeholder(tf.int32,(None, MAX_LENGTH))
        self.answers_placeholder = tf.placeholder(tf.int32, (None, MAX_LENGTH))
        #self.seqs_placeholder = tf.placeholder(tf.float32, (None, self.max_length, 2 * self.num_topics))
        self.seq_lens_placeholder = tf.placeholder(tf.int32, (None))
        self.mask_placeholder = tf.placeholder(tf.bool, (None, self.max_length))
        self.dropout_placeholder = tf.placeholder_with_default(DROPOUT, ())
        self.lr_placeholder = tf.placeholder_with_default(LR, ())

    def data_pipeline(self):
        xav_init = tf.contrib.layers.xavier_initializer()
        d = 1.0 - self.dropout_placeholder
        h = self.hidden_size

        embeddings = tf.Variable(
            tf.random_uniform([self.num_topics * 2, EMBEDDING_SIZE], -1.0, 1.0))

        #Can use this instead of seqs_placeholder
        #obs_seqs = tf.one_hot(self.topics_placeholder * 2 + self.answers_placeholder, 2*self.num_topics)
        #batch_size = tf.shape(obs_seqs)[0]
        #self.seqs = tf.concat([tf.zeros((batch_size, 1, 2*self.num_topics)), obs_seqs], 1)
        init_state1 = tf.Variable(tf.random_uniform([BATCH_SIZE, self.hidden_size], -1.0, 1.0))
        init_state2 = tf.Variable(tf.random_uniform([BATCH_SIZE, self.hidden_size], -1.0, 1.0))
        batch_size = tf.shape(self.topics_placeholder)[0]

        # add start token,
        topics_wo_start = self.topics_placeholder + self.num_topics * self.answers_placeholder
        all_topics = tf.concat((tf.zeros((batch_size, 1), dtype=tf.int32), topics_wo_start), axis=1)

        topic_seqs = tf.nn.embedding_lookup(embeddings, all_topics)
        # answers_embedding = tf.to_float(tf.reshape(self.answers_placeholder, (-1, MAX_LENGTH, 1)))
        # seqs_with_answers = tf.concat([topic_seqs, answers_embedding], 2)
        # self.seqs = tf.concat([tf.zeros((batch_size, 1, EMBEDDING_SIZE + 1)), seqs_with_answers], 1)
        self.seqs = topic_seqs

        with tf.variable_scope('lstm1'):
            cell = tf.contrib.rnn.LSTMCell(h)
            cell = tf.contrib.rnn.DropoutWrapper(cell, output_keep_prob=d)
            outputs1, hidden_states = tf.nn.dynamic_rnn(# initial_state=init_state1,
                cell=cell, inputs=self.seqs,
                sequence_length=self.seq_lens_placeholder + 1, dtype=tf.float32,
                swap_memory=True)

        with tf.variable_scope('lstm2'):
            cell2 = tf.contrib.rnn.LSTMCell(h)
            cell2 = tf.contrib.rnn.DropoutWrapper(cell2, output_keep_prob=d)

            outputs, hidden_states = tf.nn.dynamic_rnn(# initial_state=init_state2,
                cell=cell2, inputs=outputs1,
                sequence_length=self.seq_lens_placeholder + 1, dtype=tf.float32,
                swap_memory=True)

        w = tf.get_variable("W", (self.hidden_size, self.num_topics), tf.float32, xav_init)
        b = tf.get_variable("b", (self.num_topics,), tf.float32, tf.constant_initializer(0.0))
        outputs_flat = tf.reshape(outputs, [-1, self.hidden_size])
        inner = tf.matmul(outputs_flat, w) + b

        # p(correct) predicted for each time step, for each topic class
        self.all_probs = tf.sigmoid(tf.reshape(inner, [-1, self.max_length + 1, self.num_topics]))

        #Slice up the probabilities so that the last ones (for the *next*
        #questions encountered) are in a separate op
        # self.post_probs = tf.slice(self.all_probs, [0,self.seq_lens_placeholder - 1,0], [-1, -1, -1])
        self.post_probs = self.all_probs[:, self.seq_lens_placeholder[0]] # assumes all seq_lens are equal
        self.probs = tf.slice(self.all_probs, [0,0,0], [-1, self.max_length, -1])

        self.v_hats = tf.reduce_sum(self.probs, 2)

        topic_indicators = tf.one_hot(self.topics_placeholder, self.num_topics)
        self.topical_probs = tf.reduce_sum(self.probs * topic_indicators, 2)

        return self.topical_probs

    def evaluate_probs(self):
        #Probabilities of getting each class at each time correct, for reference
        self.guesses = tf.to_int32(tf.round(self.topical_probs))
        corrects = tf.to_int32(tf.equal(self.guesses, self.answers_placeholder))
        masked_corrects = tf.boolean_mask(corrects, self.mask_placeholder)
        num_correct = tf.reduce_sum(masked_corrects)

        self.auc_score, self.auc_op = tf.metrics.auc(self.answers_placeholder, self.topical_probs)

        # How many total examples in this batch - for the denominator of # correct
        self.num_total = tf.reduce_sum(tf.to_int32(self.mask_placeholder))

        return num_correct

    def find_loss(self, probs, labels):
        losses = probs - tf.to_float(labels)
        masked_losses = tf.boolean_mask(losses, self.mask_placeholder)

        #Calculate mean-squared error
        total_loss = tf.reduce_sum(masked_losses**2)
        return total_loss

    def setup_system(self):
        self.add_placeholders()

        with tf.variable_scope("dkt"):
            self.topical_probs = self.data_pipeline()
            self.num_correct = self.evaluate_probs()

        self.loss = self.find_loss(self.topical_probs, self.answers_placeholder)
        self.train_op = tf.train.AdamOptimizer(self.lr_placeholder).minimize(self.loss)

        self.saver = tf.train.Saver()

    def train_on_batch(self, session, lens_batch, masks_batch, answers_batch, topics_batch):
        feed_dict = {#self.seqs_placeholder: seqs_batch,
                     self.seq_lens_placeholder: lens_batch,
                     self.mask_placeholder: masks_batch,
                     self.answers_placeholder: answers_batch,
                     self.topics_placeholder: topics_batch}
        _, loss = session.run([self.train_op, self.loss], feed_dict=feed_dict)
        return loss

    def test_on_batch(self, session, lens_batch, masks_batch, answers_batch, topics_batch):
        feed_dict = {#self.seqs_placeholder: seqs_batch,
                     self.seq_lens_placeholder: lens_batch,
                     self.mask_placeholder: masks_batch,
                     self.answers_placeholder: answers_batch,
                     self.topics_placeholder: topics_batch}
        num_correct, num_total, auc, v_hats = \
            session.run([self.num_correct, self.num_total, self.auc_op, self.v_hats], feed_dict=feed_dict)
        return num_correct, num_total, auc, v_hats

    # Returns array of predicted next probabilities of correctness
    # Shape 1 x batch size x 1 x num_topics
    def next_probs(self, session, lens_batch, masks_batch, answers_batch, topics_batch):
        feed_dict = {
            self.seq_lens_placeholder: lens_batch,
            self.mask_placeholder: masks_batch,
            self.answers_placeholder: answers_batch,
            self.topics_placeholder: topics_batch}
        post_probs, = session.run([self.post_probs], feed_dict=feed_dict)
        return post_probs

    def next_probs_with_topical(self, session, lens_batch, masks_batch, answers_batch, topics_batch):
        feed_dict = {
            self.seq_lens_placeholder: lens_batch,
            self.mask_placeholder: masks_batch,
            self.answers_placeholder: answers_batch,
            self.topics_placeholder: topics_batch}
        post_probs, topical_probs = session.run([self.post_probs, self.topical_probs], feed_dict=feed_dict)
        return post_probs, topical_probs

    def __init__(self, num_topics, hidden_size, max_length):
        self.num_topics = num_topics
        self.hidden_size = hidden_size
        self.max_length = max_length

        self.setup_system()

#Takes list of tuples of fields eg [(seqs0, lens0), (seqs1, lens1)...]
#returns batches of form [seqs_batch, lens_batch, ...]
def batchify(data):
    num_batches = int(np.ceil(len(data) / BATCH_SIZE))
    num_fields = len(data[0])

    batches = []
    for i in range(num_batches):
        #batch up all the different sequences of data (seqs, lens, etc)
        batch_data = data[i*BATCH_SIZE:(i+1)*BATCH_SIZE]

        batch = [[] for i in range(len(batch_data[0]))]
        for b in batch_data:
            for j in range(num_fields):
                batch[j].append(b[j])
        batches.append(batch)

    return batches

CKPT_FILE = './models.ckpt'
#Trains 2 DKT models, one on each half of the data
def train_paired_models(session, data, num_topics):

    with tf.variable_scope("model1"):
        model1 = DKTModel(num_topics, HIDDEN_SIZE, MAX_LENGTH)
    with tf.variable_scope("model2"):
        model2 = DKTModel(num_topics, HIDDEN_SIZE, MAX_LENGTH)

    session.run(tf.local_variables_initializer())

    lens, masks, answers, topics = data
    assert(len(lens) == len(masks) == len(answers) == len(topics))
    cutoff = int(len(lens) * 0.5)
    zipped_data = list(zip(*data))
    first_data = zipped_data[:cutoff]
    second_data = zipped_data[cutoff:]

    saver = tf.train.Saver()
    try:
        saver.restore(session, CKPT_FILE)
        return model1, model2, second_data
    except:
        pass

    session.run(tf.global_variables_initializer())

    def train_model_epoch(model, train_data, dev_data, epoch_num, ):
        np.random.shuffle(train_data)
        train_batches = batchify(train_data)

        print("starting training epoch", epoch_num)
        for batch_num, train_batch in enumerate(train_batches):
            loss = model.train_on_batch(session, *train_batch)
            if batch_num % 10 == 0:
                print("On batch number {}, loss is {}".format(batch_num, loss))

        print("Epoch {} complete, evaluating model...".format(epoch_num))
        eval_model(dev_data, model, session)

    for epoch in range(3):
        print("First Model")
        train_model_epoch(model1, first_data, second_data, epoch)

    for epoch in range(3):
        print("Second Model")
        train_model_epoch(model2, second_data, first_data, epoch)

    saver.save(session, CKPT_FILE)

    #return model1, first_seq_lens, first_answers, first_topics, first_masks, first_v_hats, \
    #    model2, second_seq_lens, second_answers, second_topics, second_masks, second_v_hats,
    return model1, model2, second_data

def test_paired_models(session, data, model1, model2):
    zipped_data = list(zip(*data))
    cutoff = int(len(data[0]) * 0.5)
    first_data = zipped_data[:cutoff]
    second_data = zipped_data[cutoff:]

    print("Evaluating first model")
    eval_model(second_data, model1, session)
    print("Evaluating second model")
    eval_model(first_data, model2, session)

def train_model(model, session, data):
    lens, masks, answers, topics = data
    assert(len(lens) == len(masks) == len(answers) == len(topics))
    cutoff = int(len(lens) * TRAIN_SPLIT)
    zipped_data = list(zip(*data))
    train_data = zipped_data[:cutoff]
    test_data = zipped_data[cutoff:]

    for epoch in range(MAX_EPOCHS):
        np.random.shuffle(train_data)
        train_batches = batchify(train_data)

        print("Starting training epoch", epoch)
        for batch_num, train_batch in enumerate(train_batches):
            loss = model.train_on_batch(session, *train_batch)
            print("On batch number {}, loss is {}".format(batch_num, loss))

        print("Epoch {} complete, evaluating model...".format(epoch))
        eval_model(test_data, model, session)

def eval_model(test_data, model, session):
    np.random.shuffle(test_data)
    test_batches = batchify(test_data)
    total_correct = 0 #total number of examples we got right
    total_total = 0 #total number of examples we tried on
    total_auc = 0.0
    for test_batch in test_batches:
        num_correct, num_total, auc, v_hats = model.test_on_batch(session, *test_batch)
        total_auc += num_total * auc
        total_correct += num_correct
        total_total += num_total
    print("{} test examples right out of {}, which is {} percent. Mean AUC {}\n".format(
        total_correct, total_total, 100.0*total_correct/total_total, 1.0*total_auc/total_total))

def get_paired_models(session, return_second_data = False):
    topics, answers, num_topics = read_assistments_data(DATA_LOC)
    full_data = load_data(topics, answers, num_topics)

    model1, model2, second_data = train_paired_models(session, full_data, num_topics)
    #test_paired_models(session, full_data, model1, model2)

    if return_second_data: return model1, model2, second_data
    return model1, model2

def main(_):
    print('tf version', tf.__version__)

    topics, answers, num_topics = read_assistments_data(DATA_LOC)
    full_data = load_data(topics, answers, num_topics)

    model = DKTModel(num_topics, HIDDEN_SIZE, MAX_LENGTH)

    with tf.Session() as session:
        session.run(tf.global_variables_initializer())
        #We need to explicitly initialize local variables to use
        #TensorFlow's AUC function for some reason...
        session.run(tf.local_variables_initializer())
        train_model(model, session, full_data)
        #model1, model2 = train_paired_models(session, full_data, num_topics)
        #test_paired_models(session, full_data, model1, model2)
        #embed()

if __name__ == "__main__":
    tf.app.run()


import vtk
import numpy as np

from VTKInterface.Interfaces.Base_Interface import BaseInterface
from VTKInterface.Utility.Conversion_Utility import convert_numpy_array_to_vtk_transform
from VTKInterface.Utility.Conversion_Utility import convert_vtk_matrix_to_numpy_array

class CameraExtrinsicInterface(BaseInterface):

    def __init__(self, vtk_renderer=None, vtk_render_window=None):

        self.init_instance_variable(
            "vtk_renderer", vtk_renderer, vtk.vtkRenderer())
        self.init_instance_variable(
            "vtk_render_window", vtk_render_window, vtk.vtkRenderWindow())

    def print_active_camera_pose(self):
        active_vtk_camera = self.vtk_renderer.GetActiveCamera()
        print("GetPosition()", active_vtk_camera.GetPosition())
        print("GetViewUp()", active_vtk_camera.GetViewUp())
        print("GetFocalPoint()", active_vtk_camera.GetFocalPoint())

    def set_active_cam_model_view_transformation(self, np_mat):
        self.set_active_cam_model_view_transformation_to_identity()
        active_vtk_camera = self.vtk_renderer.GetActiveCamera()

        vtk_transform = convert_numpy_array_to_vtk_transform(np_mat)
        active_vtk_camera.ApplyTransform(vtk_transform)

    def set_active_cam_model_view_transformation_to_identity(self):
        active_vtk_camera = self.vtk_renderer.GetActiveCamera()

        # By default the VTK camera is shifted by (0, 0, 1), i.e.
        # active_vtk_camera.GetPosition() returns (0.0, 0.0, 1.0)

        # Option 1
        # inverse_transform = vtk.vtkTransform()
        # inverse_transform.Identity()
        # # GetModelViewTransformMatrix() returns a cam_to_world_matrix
        # # Therefore, it is already the inverse of the previous world_to_cam_matrix
        # model_view_mat = active_vtk_camera.GetModelViewTransformMatrix()
        # inverse_transform.Concatenate(model_view_mat)
        # active_vtk_camera.ApplyTransform(inverse_transform)

        # Option 2
        active_vtk_camera.SetFocalPoint(0.0, 0.0, -1.0)
        active_vtk_camera.SetViewUp(0.0, 1.0, 0.0)
        active_vtk_camera.SetPosition(0.0, 0.0, 0.0)

        model_view_trans_mat = convert_vtk_matrix_to_numpy_array(
            active_vtk_camera.GetModelViewTransformMatrix())
        assert np.array_equal(
            model_view_trans_mat,
            np.identity(4, dtype=float))

    def set_active_cam_model_view_transformation_from_opengl_cam(self,
                                                                 opengl_cam):

        # The virtual camera MUST be initialized with an OpenGl cam_to_world_mat

        active_vtk_camera = self.vtk_renderer.GetActiveCamera()
        camera_center = opengl_cam.get_camera_center()
        active_vtk_camera.SetPosition(camera_center)
        viewing_dir = opengl_cam.cam_direction_to_world_direction(
            np.array([0, 0, -1]))
        focal_point = camera_center + viewing_dir
        active_vtk_camera.SetFocalPoint(focal_point)
        # active_vtk_camera.ComputeViewPlaneNormal()
        view_up = opengl_cam.cam_direction_to_world_direction(
            np.array([0, 1, 0]))
        active_vtk_camera.SetViewUp(view_up)


########################################################################################
#                     Script author: Michał Tomaszewski                                #
########################################################################################

import numpy as np
import seaborn as sns
import sklearn.metrics as sklm
import matplotlib.pyplot as plt

import tensorflow as tf
import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Dense, Conv2D, BatchNormalization, MaxPooling2D, Flatten, Dropout
from keras.models import Sequential, load_model, Model
from keras.applications import VGG16
from keras.optimizers import Adam, SGD, RMSprop
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.applications.vgg16 import preprocess_input as vgg16_preprocessing

from glaucpack import run_tests, smooth_curve, visualize_training

class MetricsPack(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):

        self.validation_data = val_generator
        self.confusion = []
        self.precision = []
        self.recall = []
        self.f1s = []
        self.kappa = []
        self.auc = []
        self.lowest_FN = np.sum(val_generator.classes)
        self.acceptable_FP = np.round(0.5*(len(val_generator.classes) - self.lowest_FN))

    def on_epoch_end(self, epoch, logs={}):

        val_generator = self.validation_data

        batch_size = val_batch_size
        breakpoint = len(val_generator) - 1

        if type(self.model.layers[-1]) is keras.engine.sequential.Sequential:
            class_layer_config = self.model.layers[-1].layers[-1].get_config()
        else:
            class_layer_config = self.model.layers[-1].get_config()

        # different handlers for different functions in the last node
        if class_layer_config['activation'] == 'sigmoid':

            val_data = np.zeros((len(val_generator)*batch_size, 256, 256, 3))
            val_labels = np.zeros((len(val_generator)*batch_size))

            for batch_id, (x, y) in enumerate(val_generator):
                val_data[batch_id*batch_size:(batch_id+1)*batch_size] = x
                val_labels[batch_id*batch_size:(batch_id+1)*batch_size] = y

                if batch_id == breakpoint:
                    break

            score = np.asarray(self.model.predict(val_data))
            predict = np.round(np.asarray(self.model.predict(val_data)))
            targ = val_labels

            go_further = True

        elif class_layer_config['activation'] == 'softmax' and class_layer_config['units'] == 2:
            print('got here')
            val_data = np.zeros((len(val_generator)*batch_size, 256, 256, 3))
            val_labels = np.zeros((len(val_generator)*batch_size, 2))

            for batch_id, (x, y) in enumerate(val_generator):
                val_data[batch_id*batch_size:(batch_id+1)*batch_size] = x
                val_labels[batch_id*batch_size:(batch_id+1)*batch_size, :] = y

                if batch_id == breakpoint:
                    break

            score = np.asarray(self.model.predict(val_data))
            predict = np.argmax(score, axis = 1)
            targ = np.argmax(val_labels, axis = 1)

            score = np.max(score, axis = 1)

            go_further = True

        else:
            print('Only binary classification is supported.')
            go_further = False

        # calculating metrics
        if go_further:
            epoch_auc = sklm.roc_auc_score(targ, score)
            epoch_confusion = sklm.confusion_matrix(targ, predict)
            epoch_precision = sklm.precision_score(targ, predict)
            epoch_recall = sklm.recall_score(targ, predict)
            epoch_f1 = sklm.f1_score(targ, predict)

            self.auc.append(epoch_auc)
            self.confusion.append(epoch_confusion)
            self.precision.append(epoch_precision)
            self.recall.append(epoch_recall)
            self.f1s.append(epoch_f1)

            epoch_FNs = epoch_confusion.ravel()[2]
            epoch_FPs = epoch_confusion.ravel()[1]

            # saving the model with lowest FN yet
            if epoch_FNs < self.lowest_FN and epoch_FPs <= self.acceptable_FP:
                filename = self.model.layers[0].name + '_low_FN_checkpoint.hdf5'
                self.model.save(filename)

                self.lowest_FN = epoch_FNs

                print('Found new low for FN. Saving this model to {}'.format(filename))
            else:
                pass

            print('Metrics on epoch end')
            print('AUC = {}'.format(epoch_auc))
            print('Precision = {}'.format(epoch_precision))
            print('Recall = {}'.format(epoch_recall))
            print('F1 = {}'.format(epoch_f1))
            print('False negativess = {}'.format(epoch_FNs))

            # If in JupyterNotebbok this would show a heatmap every interation
            # sns.heatmap(epoch_confusion, cmap = 'coolwarm', annot = True)
            # plt.xlabel('Predicted Labels')
            # plt.ylabel('True Labels')
            # plt.show()

train_dir = r''
val_dir = r''
test_dir = r''

# datagens
aggresive_augmentation = True
if aggresive_augmentation:
    train_datagen = ImageDataGenerator(#rescale = 1/255, # not needed with vgg16_preprocessing
                                       rotation_range = 40,
                                       width_shift_range = 0.1,
                                       height_shift_range = 0.1,
                                       zoom_range = 0.1,
                                       horizontal_flip = True,
                                       preprocessing_function = vgg16_preprocessing,
                                       fill_mode = 'reflect')
else:
    train_datagen = ImageDataGenerator(#rescale = 1/255,
                                       horizontal_flip = True,
                                       vertical_flip = True,
                                       preprocessing_function = vgg16_preprocessing)

test_datagen = ImageDataGenerator(#rescale = 1/255,
                                  preprocessing_function = vgg16_preprocessing)

# MODEL DEFINITION
target_size = (256, 256)
input_shape = target_size + (3,)

transfer_learning = True
if transfer_learning:
    conv_base = VGG16(weights = 'imagenet',
                include_top = False,
                input_shape = input_shape)

    for layer in conv_base.layers:
        layer.trainable = False

else:
    conv_base = VGG16(weights = None,
                     include_top = False,
                     input_shape = input_shape)

model = Sequential()
model.add(conv_base)
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(1024, activation = 'relu'))
model.add(Dense(1024, activation = 'relu'))
model.add(Dense(1, activation = 'sigmoid'))

# prep generators
train_batch_size = 10
train_generator = train_datagen.flow_from_directory(train_dir,
                                                   target_size = target_size,
                                                   batch_size = train_batch_size,
                                                   class_mode = 'binary',
                                                   shuffle = True)

val_batch_size = 6
val_generator = test_datagen.flow_from_directory(val_dir,
                                                target_size = target_size,
                                                batch_size = val_batch_size,
                                                class_mode = 'binary')

test_generator = test_datagen.flow_from_directory(test_dir, target_size = target_size,
                                                 batch_size = 1, shuffle = False, class_mode = 'binary')

metrics = MetricsPack()

model.compile(optimizer = Adam(),
             loss = 'binary_crossentropy',
             metrics = ['acc'])

val_loss_callback = ModelCheckpoint('train_demo_test_chckpt.hdf5', monitor = 'val_loss', verbose = 1, save_best_only = True)
early_stopper = EarlyStopping(monitor = 'val_loss', patience = 10, min_delta = 0.1)

train_bool = True
if train_bool:
    history = model.fit_generator(train_generator, validation_data = val_generator,
                        epochs = 1,
                        verbose = 1,
                        callbacks = [metrics, val_loss_callback])

# UNFREEZING SOME LAYERS
for layer in model.layers[0].layers[:15]:
    layer.trainable = False
for layer in model.layers[0].layers[15:]:
    layer.trainable = True

for layer_id, layer in enumerate(model.layers[0].layers):
    print(layer_id, layer.name, layer.trainable)

# we need to recompile the model for these modifications to take effect
# we use SGD with a low learning rate
model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='binary_crossentropy', metrics = ['acc'])

val_loss_callback = ModelCheckpoint('train_demo_test_chckpt.hdf5', monitor = 'val_loss', verbose = 1, save_best_only = True)

train_bool = True
if train_bool:
    history = model.fit_generator(train_generator, validation_data = val_generator,
                                epochs = 2,
                                verbose = 1,
                                callbacks = [metrics, val_loss_callback])

# whether to use the last trained model or use a saved model that was trained earilier
model_in_memory = True
if not model_in_memory:
    model = load_model('equal_vgg16_tuning_checkpoint.hdf5')
    run_tests(model, test_generator)
else:
    run_tests(model, test_generator)

visualize_training(history.history, smoothed = True)


import requests
import re
from bs4 import BeautifulSoup

headers = {"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0"}
for i in range(1, 6):
    print("page : ", i)
    url = "https://www.amazon.co.uk/s?k=laptop&page={}&qid=1624653497&ref=sr_pg_2".format(i)

    res = requests.get(url, headers=headers)
    res.raise_for_status()
    soup = BeautifulSoup(res.text, "lxml")

    items = soup.find_all("div", attrs= {"class" : re.compile("^s-include-content-margin s-border-bottom s-latency-cf-section")})
    # print(items[0].find("span", attrs={"class" : "a-offscreen"}).get_text())

    for item in items :
        # exclude advertised items
        ad_items = item.find("span", attrs={"class" : "a-color-secondary"})
        # print(ad_items)
        if ad_items :
            if ad_items.get_text() == "Sponsored":
                print("Exculde advertised items\n")
                continue
        else :
            print(" ")

        name = item.find("h2", attrs={"class" : "a-size-mini a-spacing-none a-color-base s-line-clamp-2"}).get_text()
        if "Samsung" in name:
            print("Exclude Samsug products\n")
            continue

        price = item.find("span", attrs={"class" : "a-offscreen"})
        if price:
            price = price.get_text()
        else :
            price = "No price available"
            continue

        rate = item.find("span", attrs={"class" : "a-icon-alt"})
        if rate:
            rate = rate.get_text()[0:4]
        else :
            rate = "No review available"
            continue

        rate_cnt = item.find("span", attrs={"class" : "a-size-base"})
        if rate_cnt:
            rate_cnt = rate_cnt.get_text().replace(",","")
        else :
            rate_cnt = "not available"
            continue

        link = item.find("a", attrs={"class" : "a-link-normal a-text-normal"})["href"]

        if float(rate) >= 4.5 and int(rate_cnt) >= 100:
            print(f"Product name : {name}")
            print(f"Price : {price}")
            print(f"Rate : {rate} {rate_cnt} reviews")
            print("click to buy : {}" .format("https://www.amazon.co.uk" + link))
            print("====================================================================")


import enum
from functools import partial
import contextlib

from PySide2 import QtWidgets, QtCore, QtGui

from .conf import APP

@contextlib.contextmanager
def block_widget_signals(widget):
    widget.blockSignals(True)
    yield
    widget.blockSignals(False)

@enum.unique
class SLIDER_TYPE(enum.Enum):
    h = 0
    s = 1
    v = 2

class HSBWidget(QtWidgets.QWidget):
    SLIDER_MIN_MAX_VAL_MAP = {
        SLIDER_TYPE.h: (0, 359),
        SLIDER_TYPE.s: (0, 255),
        SLIDER_TYPE.v: (0, 255),

    }

    COLOR_CHANGED_SIGNAL = QtCore.Signal(tuple)

    def __init__(self, name, parent=None):
        super().__init__(parent=parent)
        self._name = name

        self._h = 0
        self._s = 0
        self._v = 0

        self._setup_ui()
        self._connect_signals()

    @property
    def color(self):
        return (self._h, self._s, self._v)

    @color.setter
    def color(self, color):
        self._h, self._s, self._v = color
        self._update_color_info()
        for slider, val in zip(
                [self._h_slider, self._s_slider, self._v_slider],
                list(color),
        ):
            with block_widget_signals(slider):
                slider.setValue(val)

    def _setup_ui(self):
        self.setStyleSheet('QWidget { border: none }')

        self._main_layout = QtWidgets.QVBoxLayout(self)

        self._name_layout = self._create_name_layout()
        self._main_layout.addLayout(self._name_layout)

        self._color_layout = self._create_color_layout()
        self._main_layout.addLayout(self._color_layout)

        self._update_color_info()

    def _create_name_layout(self):
        self._name_layout = QtWidgets.QHBoxLayout()

        self._name_label = QtWidgets.QLabel(self._name)
        self._name_layout.addWidget(self._name_label)

        self._name_layout.addStretch(1)

        self._color_text_label = self._create_color_text_label()
        self._name_layout.addWidget(self._color_text_label)

        return self._name_layout

    def _create_color_text_label(self):
        self._color_text_label = QtWidgets.QLabel()
        self._color_text_label.setFixedHeight(14)
        self._color_text_label.setAlignment(QtCore.Qt.AlignLeft)
        font_id = QtGui.QFontDatabase().addApplicationFont(APP.FONT_FILE_PATH)
        if font_id == -1:
            error_msg = f'Could not load font from {APP.FONT_FILE_PATH}'
            raise RuntimeError(error_msg)
        font = QtGui.QFont(APP.FONT_FAMILY)
        font.setPointSize(12)
        self._color_text_label.setFont(font)

        return self._color_text_label

    def _create_color_layout(self):
        self._color_layout = QtWidgets.QHBoxLayout()

        self._color_label = QtWidgets.QLabel()
        self._color_label.setMinimumWidth(30)
        self._color_layout.addWidget(self._color_label)

        self._slider_group_layout = self._create_slider_group_layout()
        self._color_layout.addLayout(self._slider_group_layout)

        return self._color_layout

    def _create_slider_group_layout(self):
        self._slider_group_layout = QtWidgets.QVBoxLayout()
        widgets = self._create_slider_layout(slider_type=SLIDER_TYPE.h)
        self._h_layout, self._h_label, self._h_slider = widgets
        widgets = self._create_slider_layout(slider_type=SLIDER_TYPE.s)
        self._s_layout, self._s_label, self._s_slider = widgets
        widgets = self._create_slider_layout(slider_type=SLIDER_TYPE.v)
        self._v_layout, self._v_label, self._v_slider = widgets
        for t in [self._h_layout, self._s_layout, self._v_layout]:
            self._slider_group_layout.addLayout(t)

        return self._slider_group_layout

    def _connect_signals(self):
        self._h_slider.valueChanged.connect(
            partial(
                self._value_changed,
                slider_type=SLIDER_TYPE.h
            )
        )

        self._s_slider.valueChanged.connect(
            partial(
                self._value_changed,
                slider_type=SLIDER_TYPE.s
            )
        )

        self._v_slider.valueChanged.connect(
            partial(
                self._value_changed,
                slider_type=SLIDER_TYPE.v
            )
        )

    def _value_changed(self, val, slider_type):
        if slider_type == SLIDER_TYPE.h:
            self._h = val
        elif slider_type == SLIDER_TYPE.s:
            self._s = val
        elif slider_type == SLIDER_TYPE.v:
            self._v = val
        else:
            error_msg = f'SLIDER TYPE {slider_type} is not defined!!'
            raise ValueError(error_msg)

        self._update_color_info()

        self.COLOR_CHANGED_SIGNAL.emit((self._h, self._s, self._v))

    def _get_color_style_string(self):
        return(
            'QWidget { background-color: '
            f'hsv({self._h}, {self._s}, {self._v})}}'
        )

    def _get_color_text_string(self):
        return(
            f'[{str(self._h).zfill(3)}, '
            f'{str(self._s).zfill(3)}, '
            f'{str(self._v).zfill(3)}]'
        )

    def _update_color_info(self):
        color_style = self._get_color_style_string()
        self._color_label.setStyleSheet(color_style)
        color_text = self._get_color_text_string()
        self._color_text_label.setText(color_text)

    def _create_slider(self, slider_type=SLIDER_TYPE.h):
        min_val, max_val = self.SLIDER_MIN_MAX_VAL_MAP.get(slider_type)
        default_val = int((max_val - min_val) / 2)
        slider = QtWidgets.QSlider(QtCore.Qt.Horizontal)
        slider.setMinimumWidth(APP.SLIDER_WIDTH)
        slider.setMaximumHeight(APP.SLIDER_HEIGHT)
        slider.setMinimum(min_val)
        slider.setMaximum(max_val)
        slider.setSingleStep(1)

        with block_widget_signals(slider):
            slider.setValue(default_val)
            if slider_type == SLIDER_TYPE.h:
                self._h = default_val
            elif slider_type == SLIDER_TYPE.s:
                self._s = default_val
            elif slider_type == SLIDER_TYPE.v:
                self._v = default_val
            else:
                error_msg = f'SLIDER TYPE {slider_type} is not defined!!'
                raise ValueError(error_msg)

        return slider

    def _create_slider_layout(self, slider_type=SLIDER_TYPE.h):
        layout = QtWidgets.QHBoxLayout()
        label = QtWidgets.QLabel(str(slider_type.name).upper())
        label.setFixedHeight(12)
        slider = self._create_slider(slider_type=slider_type)
        layout.addWidget(label)
        layout.addWidget(slider)
        return layout, label, slider

class ColorBandWidget(QtWidgets.QFrame):
    LOW_COLOR_CHANGED_SIGNAL = QtCore.Signal(tuple)
    HIGH_COLOR_CHANGED_SIGNAL = QtCore.Signal(tuple)

    def __init__(self, name, parent=None):
        super().__init__(parent=parent)
        self._name = name

        self._low_color = None
        self._high_color = None

        self._setup_ui()
        self._connect_signals()

    @property
    def low_color(self):
        return self._low_widget.color

    @low_color.setter
    def low_color(self, color):
        self._low_widget.color = color

    @property
    def high_color(self):
        return self._high_widget.color

    @high_color.setter
    def high_color(self, color):
        self._high_widget.color = color

    def _setup_ui(self):
        self.setStyleSheet('QWidget { border: 1px solid #5A5A5A }')
        self._label = QtWidgets.QLabel(self._name)
        self._label.setStyleSheet('QWidget { border: none }')

        self._main_layout = QtWidgets.QVBoxLayout(self)
        self._low_widget = HSBWidget(name='LOW', parent=self)
        self._high_widget = HSBWidget(name='HIGH', parent=self)

        self._main_layout.addWidget(self._label)
        self._main_layout.addWidget(self._low_widget)
        self._main_layout.addWidget(self._high_widget)

    def _connect_signals(self):
        self._low_widget.COLOR_CHANGED_SIGNAL.connect(self._low_color_changed)
        self._high_widget.COLOR_CHANGED_SIGNAL.connect(
            self._high_color_changed
        )

    def _low_color_changed(self, color):
        self.LOW_COLOR_CHANGED_SIGNAL.emit(color)

    def _high_color_changed(self, color):
        self.HIGH_COLOR_CHANGED_SIGNAL.emit(color)


# -*- coding: utf-8 -*-
import re
from decimal import Decimal
from datetime import timedelta
from datetime import datetime
from .log import logger as log
_author_ = 'luwt'
_date_ = '2021/9/1 16:20'

# 执行策略里，变量必须和convert_type函数的参数名保持一致
need_convert_types = {
    datetime: 'str(value)',
    # decimal 转为字符串形式
    Decimal: 'float(value)',
}

def convert_type(value):
    """将一些特殊类型转换为字符串，避免插入数据库时，参数错误"""
    result = value
    for data_type, deal_strategy in need_convert_types.items():
        if isinstance(value, data_type):
            result = eval(deal_strategy)
            break
    return result

def get_data(cursor, sql):
    cursor.execute(sql)
    return cursor.fetchall()

def get_pattern_matches(table_names, pattern, pattern_name):
    if not pattern:
        return set()
    log.info(f"{pattern_name} pattern is {pattern}")
    result = set(filter(lambda x: re.fullmatch(pattern, x), table_names))
    log.info(f"{pattern_name} match {len(result)} tables: {result}")
    return result

def times(f):
    def count_time(*args, **kw):
        start = datetime.now()
        log.info(f"开始时间为：{start}")
        try:
            f(*args, **kw)
        finally:
            end = datetime.now()
            interval = (end.timestamp() - start.timestamp())
            time_sec = timedelta(seconds=interval)
            log.info(f"结束时间：{end}，耗时{time_sec}")
    return count_time


# import libraries here
import numpy as np
import cv2 # OpenCV biblioteka
import matplotlib
import matplotlib.pyplot as plt

matplotlib.rcParams['figure.figsize'] = 10,7

def count_blood_cells(image_path):
    """
    Procedura prima putanju do fotografije i vraca broj crvenih krvnih zrnaca, belih krvnih zrnaca i
    informaciju da li pacijent ima leukemiju ili ne, na osnovu odnosa broja krvnih zrnaca

    Ova procedura se poziva automatski iz main procedure i taj deo kod nije potrebno menjati niti implementirati.

    :param image_path: <String> Putanja do ulazne fotografije.
    :return: <int>  Broj prebrojanih crvenih krvnih zrnaca,
             <int> broj prebrojanih belih krvnih zrnaca,
             <bool> da li pacijent ima leukemniju (True ili False)
    """
    red_blood_cell_count = 0
    white_blood_cell_count = 0
    has_leukemia = None

    img_rgb = cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)
    plt.imshow(img_rgb, 'gray')
    plt.show()

    img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_RGB2GRAY)
    #img_gray = cv2.GaussianBlur(img_gray, (5, 5), 0)

    #img_bin = cv2.adaptiveThreshold(img_gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 11, 2)
    #ret, img_bin = cv2.threshold(img_gray, 0, 255, cv2.THRESH_OTSU)
    ret, img_bin = cv2.threshold(img_gray, 150, 255, cv2.THRESH_BINARY)
    img_bitw = cv2.bitwise_not(img_bin)

    img_open = open(img_bitw)
    img_close = close(img_bin)

    img_er = er(img_bin)
    img_dil = dil(img_bin)

    white_blood_cell_count = len(find_white_blood_cells(img_close))
    print('BELIH: '+  str(white_blood_cell_count))
    #plt.imshow(img_close, 'gray')
    #plt.show()

    # CRVENE
    img_gray = np.ndarray((img_rgb.shape[0], img_rgb.shape[1]))
    img_gray = 0 * img_rgb[:, :, 0] + 0.1 * img_rgb[:, :, 1] + 0.9 * img_rgb[:, :, 2]
    img_gray = img_gray.astype('uint8')
    ret, img_bin = cv2.threshold(img_gray, 0, 255, cv2.THRESH_OTSU)

    img_bitw = cv2.bitwise_not(img_bin)
    img_open = open(img_bitw)

    red_blood_cell_count = len(find_red_blood_cells(img_open))
    print('CRVENIH: ' + str(red_blood_cell_count))
    #plt.imshow(img_open, 'gray')
    #plt.show()

    if(white_blood_cell_count * 12 > red_blood_cell_count):
        has_leukemia = True
    else:
        has_leukemia = False

    # TODO - Prebrojati crvena i bela krvna zrnca i vratiti njihov broj kao povratnu vrednost ove procedure

    # TODO - Odrediti da li na osnovu broja krvnih zrnaca pacijent ima leukemiju i vratiti True/False kao povratnu vrednost ove procedure

    return red_blood_cell_count, white_blood_cell_count, has_leukemia

def find_white_blood_cells(img):
    white_blood_cells = []
    img, contours, hierarchy = cv2.findContours(img, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)

    white_cells_temp = []
    for contour in contours:
        area = cv2.contourArea(contour)  # oblast konture
        x, y, width, height = cv2.boundingRect(contour)
        if width > 40 and height > 40:
            white_cells_temp.append(contour)

    if (len(white_cells_temp) <= 3):
        img_er = er(img)
        img, contours, hierarchy = cv2.findContours(img_er, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
        for contour in contours:
            area = cv2.contourArea(contour)  # oblast konture
            x, y, width, height = cv2.boundingRect(contour)
            if width > 120 and height > 120 and area > 18000:
                white_blood_cells.append(contour)
        #plt.imshow(img_er, 'gray')
        #plt.show()
    elif (len(white_cells_temp) >= 10):
        img_wat = watershed(255 - img)
        img_inv = 255 - img_wat
        img, contours, hierarchy = cv2.findContours(img_inv, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
        for contour in contours:
            white_blood_cells.append(contour)
        #plt.imshow(img_inv, 'gray')
        #plt.show()
    else:
        white_blood_cells = white_cells_temp

    return white_blood_cells

def find_red_blood_cells(img):
    pom = 0
    red_blood_cells = []
    img, contours, hierarchy = cv2.findContours(img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    for contour in contours:
        area = cv2.contourArea(contour)  # oblast konture
        if area > 220 and hierarchy[0, pom, 3] == -1:
            red_blood_cells.append(contour)
        pom += 1
    return red_blood_cells

def open(img):
    kernel_ero = np.ones((3, 3))
    kernel_dil = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))

    img_erode = cv2.erode(img, kernel_dil, iterations=1)

    img_open = cv2.dilate(img_erode, kernel_dil, iterations=1)

    return img_open

def close(img):
    kernel_ero = np.ones((3, 3))
    kernel_dil = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9, 9))

    img_dilate = cv2.dilate(img, kernel_dil, iterations=1)

    img_close = cv2.erode(img_dilate, kernel_dil, iterations=2)

    return img_close

def er(img):
    kernel_ero = np.ones((3, 3))
    img_erode = cv2.erode(img, kernel_ero, iterations=2)

    return img_erode

def dil(img):
    kernel_dil = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
    img_dil = cv2.dilate(img, kernel_dil, iterations=2)
    return img_dil

def watershed(img):
    kernel = np.ones((3, 3), np.uint8)
    opening = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel, iterations=2)
    # sure background area
    sure_bg = cv2.dilate(opening, kernel, iterations=3)

    # Finding sure foreground area
    dist_transform = cv2.distanceTransform(opening, cv2.DIST_L2, 5)
    ret, sure_fg = cv2.threshold(dist_transform, 0.6 * dist_transform.max(), 255, 0)

    # Finding unknown region
    sure_fg = np.uint8(sure_fg)
    unknown = cv2.subtract(sure_bg, sure_fg)

    '''# Marker labelling
    ret, markers = cv2.connectedComponents(sure_fg)
    # Add one to all labels so that sure background is not 0, but 1
    markers = markers + 1
    # Now, mark the region of unknown with zero
    markers[unknown == 255] = 0

    markers = cv2.watershed(img_bin, markers)
    img_bin[markers == -1] = [255, 0, 0]'''

    return sure_fg

Mini projekat 1 - Obrada slike/sc_2020_challenge_1/process.py


# Proyecto Optimización G54

from gurobipy import Model, GRB, quicksum
#Agregué el conjunto pacientes a cojuntos
from conjuntos import (centros, periodos, prestaciones, bases,
                       ambulancias, i_prestaciones,
                       ambulancias_por_base, pacientes)

# Al psi le puse una f
from parametros import (k_a, i_hf, c_pg, u_h, d_bgt, f_hgt, m_bht, r_ab,
                        lambda_pf, v_p, l_p)

modelo = Model("Sistema de Atención Médica SAMU")

# Creación de variables ----------------------------------------------

x = {}

for periodo in periodos:
    for ambulancia in ambulancias:
        for paciente in pacientes:
            name_ = f"x_{periodo}_{ambulancia}_{paciente}"
            x[periodo, ambulancia, paciente] = modelo.addVar(0,
                                                             1,
                                                             vtype=GRB.BINARY,
                                                             name=name_)

y = {}

for periodo in periodos:
    for ambulancia in ambulancias:
        for paciente in pacientes:
            for centro in centros:
                name_ = f"y_{periodo}_{ambulancia}_{paciente}_{centro}"
                y[periodo, ambulancia,
                  paciente, centro] = modelo.addVar(0, 1, vtype=GRB.BINARY,
                                                    name=name_)

s = {}

for periodo in periodos:
    for ambulancia in ambulancias:
        name_ = f"s_{periodo}_{ambulancia}"
        s[periodo, ambulancia] = modelo.addVar(0, 1, vtype=GRB.BINARY,
                                               name=name_)

# Agregar variables ----------------------------------------------

modelo.update()

# Agregar restricciones, están en el orden del informe ----------------------------------------------

#1 Una ambulancia por emergencia