import json
import multiprocessing as mp
from typing import Set, Union

import pytest

from determined.common import api
from determined.common.api import authentication, certs
from tests import config as conf
from tests import experiment as exp

@pytest.mark.e2e_cpu
@pytest.mark.timeout(600)
def test_streaming_metrics_api() -> None:
    # TODO: refactor tests to not use cli singleton auth.
    certs.cli_cert = certs.default_load(conf.make_master_url())
    authentication.cli_auth = authentication.Authentication(conf.make_master_url(), try_reauth=True)

    pool = mp.pool.ThreadPool(processes=7)

    experiment_id = exp.create_experiment(
        conf.fixtures_path("mnist_pytorch/adaptive_short.yaml"),
        conf.tutorials_path("mnist_pytorch"),
    )
    # To fully test the streaming APIs, the requests need to start running immediately after the
    # experiment, and then stay open until the experiment is complete. To accomplish this with all
    # of the API calls on a single experiment, we spawn them all in threads.

    # The HP importance portion of this test is commented out until the feature is enabled by
    # default

    metric_names_thread = pool.apply_async(request_metric_names, (experiment_id,))
    train_metric_batches_thread = pool.apply_async(request_train_metric_batches, (experiment_id,))
    valid_metric_batches_thread = pool.apply_async(request_valid_metric_batches, (experiment_id,))
    train_trials_snapshot_thread = pool.apply_async(request_train_trials_snapshot, (experiment_id,))
    valid_trials_snapshot_thread = pool.apply_async(request_valid_trials_snapshot, (experiment_id,))
    train_trials_sample_thread = pool.apply_async(request_train_trials_sample, (experiment_id,))
    valid_trials_sample_thread = pool.apply_async(request_valid_trials_sample, (experiment_id,))

    metric_names_results = metric_names_thread.get()
    train_metric_batches_results = train_metric_batches_thread.get()
    valid_metric_batches_results = valid_metric_batches_thread.get()
    train_trials_snapshot_results = train_trials_snapshot_thread.get()
    valid_trials_snapshot_results = valid_trials_snapshot_thread.get()
    train_trials_sample_results = train_trials_sample_thread.get()
    valid_trials_sample_results = valid_trials_sample_thread.get()

    if metric_names_results is not None:
        pytest.fail("metric-names: %s. Results: %s" % metric_names_results)
    if train_metric_batches_results is not None:
        pytest.fail("metric-batches (training): %s. Results: %s" % train_metric_batches_results)
    if valid_metric_batches_results is not None:
        pytest.fail("metric-batches (validation): %s. Results: %s" % valid_metric_batches_results)
    if train_trials_snapshot_results is not None:
        pytest.fail("trials-snapshot (training): %s. Results: %s" % train_trials_snapshot_results)
    if valid_trials_snapshot_results is not None:
        pytest.fail("trials-snapshot (validation): %s. Results: %s" % valid_trials_snapshot_results)
    if train_trials_sample_results is not None:
        pytest.fail("trials-sample (training): %s. Results: %s" % train_trials_sample_results)
    if valid_trials_sample_results is not None:
        pytest.fail("trials-sample (validation): %s. Results: %s" % valid_trials_sample_results)

@pytest.mark.distributed
@pytest.mark.timeout(1800)
def test_hp_importance_api() -> None:
    certs.cli_cert = certs.default_load(conf.make_master_url())
    authentication.cli_auth = authentication.Authentication(conf.make_master_url(), try_reauth=True)

    pool = mp.pool.ThreadPool(processes=1)

    experiment_id = exp.create_experiment(
        conf.fixtures_path("mnist_pytorch/random.yaml"),
        conf.tutorials_path("mnist_pytorch"),
    )

    hp_importance_thread = pool.apply_async(request_hp_importance, (experiment_id,))

    hp_importance_results = hp_importance_thread.get()

    if hp_importance_results is not None:
        pytest.fail("hyperparameter-importance: %s. Results: %s" % hp_importance_results)

def request_metric_names(experiment_id):  # type: ignore
    response = api.get(
        conf.make_master_url(),
        "api/v1/experiments/{}/metrics-stream/metric-names".format(experiment_id),
        params={"period_seconds": 1},
    )
    results = [message["result"] for message in map(json.loads, response.text.splitlines())]

    # First let's verify an empty response was sent back before any real work was done
    if results[0]["searcherMetric"] != "validation_loss":
        return ("unexpected searcher metric in first response", results)
    if results[0]["trainingMetrics"] != []:
        return ("unexpected training metric in first response", results)
    if results[0]["validationMetrics"] != []:
        return ("unexpected validation metric in first response", results)

    # Then we verify that all expected responses are eventually received exactly once
    accumulated_training = set()
    accumulated_validation = set()
    for i in range(1, len(results)):
        for training in results[i]["trainingMetrics"]:
            if training in accumulated_training:
                return ("training metric appeared twice", results)
            accumulated_training.add(training)
        for validation in results[i]["validationMetrics"]:
            if validation in accumulated_validation:
                return ("training metric appeared twice", results)
            accumulated_validation.add(validation)

    if accumulated_training != {"loss"}:
        return ("unexpected set of training metrics", results)
    if accumulated_validation != {"validation_loss", "accuracy"}:
        return ("unexpected set of validation metrics", results)
    return None

def request_train_metric_batches(experiment_id):  # type: ignore
    response = api.get(
        conf.make_master_url(),
        "api/v1/experiments/{}/metrics-stream/batches".format(experiment_id),
        params={"metric_name": "loss", "metric_type": "METRIC_TYPE_TRAINING", "period_seconds": 1},
    )
    results = [message["result"] for message in map(json.loads, response.text.splitlines())]

    # First let's verify an empty response was sent back before any real work was done
    if results[0]["batches"] != []:
        return ("unexpected batches in first response", results)

    # Then we verify that all expected responses are eventually received exactly once
    accumulated = set()
    for i in range(1, len(results)):
        for batch in results[i]["batches"]:
            if batch in accumulated:
                return ("batch appears twice", results)
            accumulated.add(batch)
    if accumulated != {100, 200, 300, 400}:
        return ("unexpected set of batches", results)
    return None

def request_valid_metric_batches(experiment_id):  # type: ignore
    response = api.get(
        conf.make_master_url(),
        "api/v1/experiments/{}/metrics-stream/batches".format(experiment_id),
        params={
            "metric_name": "accuracy",
            "metric_type": "METRIC_TYPE_VALIDATION",
            "period_seconds": 1,
        },
    )
    results = [message["result"] for message in map(json.loads, response.text.splitlines())]

    # First let's verify an empty response was sent back before any real work was done
    if results[0]["batches"] != []:
        return ("unexpected batches in first response", results)

    # Then we verify that all expected responses are eventually received exactly once
    accumulated = set()
    for i in range(1, len(results)):
        for batch in results[i]["batches"]:
            if batch in accumulated:
                return ("batch appears twice", results)
            accumulated.add(batch)
    if accumulated != {200, 400}:
        return ("unexpected set of batches", results)
    return None

def validate_hparam_types(hparams: dict) -> Union[None, str]:
    for hparam in ["dropout1", "dropout2", "learning_rate"]:
        if type(hparams[hparam]) != float:
            return "hparam %s of unexpected type" % hparam
    for hparam in ["global_batch_size", "n_filters1", "n_filters2"]:
        if type(hparams[hparam]) != int:
            return "hparam %s of unexpected type" % hparam
    return None

def request_train_trials_snapshot(experiment_id):  # type: ignore
    response = api.get(
        conf.make_master_url(),
        "api/v1/experiments/{}/metrics-stream/trials-snapshot".format(experiment_id),
        params={
            "metric_name": "loss",
            "metric_type": "METRIC_TYPE_TRAINING",
            "batches_processed": 100,
            "period_seconds": 1,
        },
    )
    results = [message["result"] for message in map(json.loads, response.text.splitlines())]

    # First let's verify an empty response was sent back before any real work was done
    if results[0]["trials"] != []:
        return ("unexpected trials in first response", results)

    # Then we verify that we receive the expected number of trials and the right types
    trials = set()
    for i in range(1, len(results)):
        for trial in results[i]["trials"]:
            trials.add(trial["trialId"])
            validate_hparam_types(trial["hparams"])
            if type(trial["metric"]) != float:
                return ("metric of unexpected type", results)
    if len(trials) != 5:
        return ("unexpected number of trials received", results)
    return None

def request_valid_trials_snapshot(experiment_id):  # type: ignore
    response = api.get(
        conf.make_master_url(),
        "api/v1/experiments/{}/metrics-stream/trials-snapshot".format(experiment_id),
        params={
            "metric_name": "accuracy",
            "metric_type": "METRIC_TYPE_VALIDATION",
            "batches_processed": 200,
            "period_seconds": 1,
        },
    )
    results = [message["result"] for message in map(json.loads, response.text.splitlines())]

    # First let's verify an empty response was sent back before any real work was done
    if results[0]["trials"] != []:
        return ("unexpected trials in first response", results)

    # Then we verify that we receive the expected number of trials and the right types
    trials = set()
    for i in range(1, len(results)):
        for trial in results[i]["trials"]:
            trials.add(trial["trialId"])
            hparam_error = validate_hparam_types(trial["hparams"])
            if hparam_error is not None:
                return (hparam_error, results)
            if type(trial["metric"]) != float:
                return ("metric of unexpected type", results)
    if len(trials) != 5:
        return ("unexpected number of trials received", results)
    return None

def check_trials_sample_result(results: list) -> Union[None, tuple]:
    # First let's verify an empty response was sent back before any real work was done
    if (
        results[0]["trials"] != []
        or results[0]["promotedTrials"] != []
        or results[0]["demotedTrials"] != []
    ):
        return ("unexpected trials in first response", results)

    # Then we verify that we receive the expected number of trials and the right types
    trials: Set[int] = set()
    datapoints = {}
    for i in range(1, len(results)):
        newTrials = set()
        for trial in results[i]["promotedTrials"]:
            if trial in trials:
                return ("trial lists as promoted twice", results)
            newTrials.add(trial)
            datapoints[trial] = 0
        for trial in results[i]["trials"]:
            if trial["trialId"] in newTrials:
                hparam_error = validate_hparam_types(trial["hparams"])
                if hparam_error is not None:
                    return (hparam_error, results)
            else:
                if trial["hparams"] is not None:
                    return ("hparams repeated for trial", results)
            for point in trial["data"]:
                if point["batches"] > datapoints[trial["trialId"]]:
                    datapoints[trial["trialId"]] = point["batches"]
                else:
                    return ("data received out of order: " + str(trial["trialId"]), results)
    return None

def request_train_trials_sample(experiment_id):  # type: ignore
    response = api.get(
        conf.make_master_url(),
        "api/v1/experiments/{}/metrics-stream/trials-sample".format(experiment_id),
        params={
            "metric_name": "loss",
            "metric_type": "METRIC_TYPE_TRAINING",
            "period_seconds": 1,
        },
    )
    results = [message["result"] for message in map(json.loads, response.text.splitlines())]
    return check_trials_sample_result(results)

def request_valid_trials_sample(experiment_id):  # type: ignore
    response = api.get(
        conf.make_master_url(),
        "api/v1/experiments/{}/metrics-stream/trials-sample".format(experiment_id),
        params={
            "metric_name": "accuracy",
            "metric_type": "METRIC_TYPE_VALIDATION",
            "period_seconds": 1,
        },
    )
    results = [message["result"] for message in map(json.loads, response.text.splitlines())]
    return check_trials_sample_result(results)

def request_hp_importance(experiment_id):  # type: ignore
    response = api.get(
        conf.make_master_url(),
        "api/v1/experiments/{}/hyperparameter-importance".format(experiment_id),
        params={"period_seconds": 1},
    )
    results = [message["result"] for message in map(json.loads, response.text.splitlines())]

    lastResult = results[-1]
    if len(lastResult["trainingMetrics"]) != 1 or len(lastResult["validationMetrics"]) != 1:
        return ("Unexpected number of metrics", lastResult)

    def valid_importance(x: float) -> bool:
        return x >= 0 and x <= 1

    loss = lastResult["trainingMetrics"]["loss"]
    searcherMetric = lastResult["validationMetrics"]["validation_loss"]

    for metric in [loss, searcherMetric]:
        if not metric["error"] == "":
            return ("Unexpected error in HP importance", lastResult)
        if metric["pending"] or metric["inProgress"]:
            return ("Unexpected incomplete status in HP importance", lastResult)
        if not metric["experimentProgress"] == 1:
            return ("HP importance from unfinished experiment included!", lastResult)
        for hparam in [
            "dropout1",
            "dropout2",
            "learning_rate",
            "n_filters1",
            "n_filters2",
        ]:
            if hparam not in metric["hpImportance"]:
                return ("Missing hparams %s" % hparam, lastResult)
            if not valid_importance(metric["hpImportance"][hparam]):
                return ("Unexpected importance for hparam %s" % hparam, lastResult)
    return None


from typing import List

from feast import Entity
from feast.feature_view import FeatureView
from feast.value_type import ValueType

def infer_entity_value_type_from_feature_views(
    entities: List[Entity], feature_views: List[FeatureView]
) -> List[Entity]:
    """
    Infer entity value type by examining schema of feature view input sources
    """
    incomplete_entities = {
        entity.name: entity
        for entity in entities
        if entity.value_type == ValueType.UNKNOWN
    }
    incomplete_entities_keys = incomplete_entities.keys()

    for view in feature_views:
        if not (incomplete_entities_keys & set(view.entities)):
            continue  # skip if view doesn't contain any entities that need inference

        col_names_and_types = view.input.get_table_column_names_and_types()
        for entity_name in view.entities:
            if entity_name in incomplete_entities:
                # get entity information from information extracted from the view input source
                extracted_entity_name_type_pairs = list(
                    filter(lambda tup: tup[0] == entity_name, col_names_and_types)
                )
                if len(extracted_entity_name_type_pairs) == 0:
                    # Doesn't mention inference error because would also be an error without inferencing
                    raise ValueError(
                        f"""No column in the input source for the {view.name} feature view matches
                        its entity's name."""
                    )

                entity = incomplete_entities[entity_name]
                inferred_value_type = view.input.source_datatype_to_feast_value_type()(
                    extracted_entity_name_type_pairs[0][1]
                )

                if (
                    entity.value_type != ValueType.UNKNOWN
                    and entity.value_type != inferred_value_type
                ) or (len(extracted_entity_name_type_pairs) > 1):
                    raise ValueError(
                        f"""Entity value_type inference failed for {entity_name} entity.
                        Multiple viable matches. Please explicitly specify the entity value_type
                        for this entity."""
                    )

                entity.value_type = inferred_value_type

    return entities


import random

batas = int(input("Batas angka yang akan ditebak : "))
def tebakAngka(x):
    angka_acak = random.randint(1, x)
    tebak = 0
    guess = 0
    while tebak != angka_acak:
        tebak = int(input("Tebaklah angka yang berada di antara angka 1 hingga {} : ".format(x)))
        if tebak > batas:
            print("Maaf, angka yang anda inputkan lebih besar dari batas \n")
        elif tebak > angka_acak:
            print("Maaf, angka anda terlalu tinggi \n")
            guess += 1
        elif tebak < angka_acak:
            print("Maaf, angka anda terlalu rendah \n")
            guess += 1
    guess += 1
    print("Horee.. selamat anda berhasil menebak angka {}".format(angka_acak))
    print("Anda berhasil menebak sebanyak {} kali".format(guess))

tebakAngka(batas)


# taken from https://github.com/pytorch/examples/blob/master/mnist/main.py
import argparse
import torch
from torch import nn
from torch.nn import functional as F
from torch import optim
from torchvision import datasets, transforms
from torch.autograd import Variable

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(320, 50)
        self.fc2 = nn.Linear(50, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 320)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

def train(args, kwargs):
    torch.manual_seed(args.seed)
    if args.cuda:
        torch.cuda.manual_seed(args.seed)

    train_loader = torch.utils.data.DataLoader(
        datasets.MNIST('./data', train=True, download=True,
                       transform=transforms.Compose([
                           transforms.ToTensor(),
                           transforms.Normalize((0.1307,), (0.3081,))
                       ])),
        batch_size=args.batch_size, shuffle=True, **kwargs)

    model = Net()
    if args.cuda:
        model.cuda()

    optimizer = optim.SGD(model.parameters(), lr=args.lr,
                          momentum=args.momentum)

    for epoch in range(1, args.epochs+1):
        model.train()
        for batch_idx, (data, target) in enumerate(train_loader):
            if args.cuda:
                data, target = data.cuda(), target.cuda()
            data, target = Variable(data), Variable(target)
            optimizer.zero_grad()
            output = model(data)
            loss = F.nll_loss(output, target)
            loss.backward()
            optimizer.step()
            if batch_idx % args.log_interval == 0:
                print('Train Epoch: {}[{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                    epoch, batch_idx*len(data), len(train_loader.dataset),
                    100. * batch_idx/len(train_loader), loss.data[0]))

    return model

def test(args, kwargs, model):
    model.eval()
    test_loss = 0
    correct = 0
    test_loader = torch.utils.data.DataLoader(
        datasets.MNIST('./data', train=False, transform=transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,))
        ])),
        batch_size=args.test_batch_size, shuffle=True, **kwargs)
    for data, target in test_loader:
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data, volatile=True), Variable(target)
        output = model(data)
        test_loss += F.nll_loss(output, target, size_average=False).data[0]
        pred = output.data.max(1, keepdim=True)[1]
        correct += pred.eq(target.data.view_as(pred)).long().cpu().sum()

    test_loss /= len(test_loader.dataset)
    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))

def parse_argument():
    parser = argparse.ArgumentParser(description="PyTorch MNIST Example")
    parser.add_argument('--batch_size', type=int, default=64,
                        metavar='N', help='input batch size for training')
    parser.add_argument('--test_batch_size', type=int, default=1000,
                        metavar='N', help='input batch size for testinf')
    parser.add_argument('--epochs', type=int, default=10,
                        metavar='N', help='number of epochs to train')
    parser.add_argument('--lr', type=float, default=0.01,
                        metavar='LR', help='learning rate')
    parser.add_argument('--momentum', type=float, default=0.5,
                        metavar='M', help='SGD momentum')
    parser.add_argument('--no-cuda', action='store_true',
                        default=False, help='disable CUDA training')
    parser.add_argument('--seed', type=int, default=1,
                        metavar='S', help='random seed')
    parser.add_argument('--log_interval', type=int, default=10, metavar='N',
                        help='how many batches to wait before logging training status')
    args = parser.parse_args()
    args.cuda = not args.no_cuda and torch.cuda.is_available()
    return args

def main():
    args = parse_argument()
    kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
    trained_model = train(args, kwargs)
    test(args, kwargs, trained_model)

if __name__ == '__main__':
    main()


# First and Last 3 characters of a string

# option-1
string = 'Wonderful'
if len(string) < 6:
    print('')
else:
    new_string = string[:3] + string[len(string)-3:]
    print(new_string)

# option-2
word = 'Amazing'
num_char = 3
if len(word) < num_char*2:
    print('')
else:
    answer = word[:num_char] + word[len(word)-num_char:]
    print(answer)


import graphene
from flask import g

from stronk.schemas.workout.type import Workout
from stronk.schemas.workout_exercise.type import WorkoutExercise as WorkoutExercise

from stronk.models.program import Program as ProgramModel
from stronk.models.program_workouts import ProgramWorkouts as ProgramWorkoutsModel
from stronk.models.workout import Workout as WorkoutModel
from stronk.models.workout_exercise import WorkoutExercise as WorkoutExerciseModel
from stronk.utils.date import date_time_str_to_date
from stronk.utils.auth import is_authorized

class CreateWorkout(graphene.Mutation):
    """ creates a workout and adds it to a program - result is the new workout
    """
    # mutation results
    workout = graphene.Field(Workout)

    class Arguments:
        name = graphene.String(required=True)
        description = graphene.String(required=False)
        projected_time = graphene.Int(required=False)
        program_id = graphene.Int(required=True)
        scheduled_time = graphene.String(required=True)

    def mutate(root, info, name: str, program_id: int, scheduled_time: str, description: str = None, projected_time: int = None):
        program = ProgramModel.find_by_id(program_id)
        # convert date time string to datetime object
        formatted_time = date_time_str_to_date(scheduled_time)

        workout = WorkoutModel.create(author=g.id,
                                      name=name,
                                      description=description,
                                      projected_time=projected_time,
                                      scheduled_time=formatted_time)

        program_workout = ProgramWorkoutsModel.create(program_id=program.id,
                                                      workout_id=workout.id)
        return CreateWorkout(workout=workout)

class UpdateWorkout(graphene.Mutation):
    """ updates a workout's information
    """
    # mutation results
    workout = graphene.Field(Workout)

    class Arguments:
        id = graphene.Int(required=True)
        name = graphene.String(required=False)
        description = graphene.String(required=False)
        projected_time = graphene.Int(required=False)
        scheduled_time = graphene.String(required=False)

    def mutate(root, info, id: str, name: str = None, description: str = None, projected_time: int = None, scheduled_time: str = None):
        workout = WorkoutModel.find_by_id(id)
        is_authorized(workout.author)

        attrs = {}
        if name:
            attrs['name'] = name
        if description:
            attrs['description'] = description
        if projected_time:
            attrs['projected_time'] = projected_time
        if scheduled_time:
            attrs['scheduled_time'] = date_time_str_to_date(scheduled_time)

        workout.update(attrs)

        return UpdateWorkout(workout=workout)

class DeleteWorkout(graphene.Mutation):
    """ removes a workout from a program
    """
    # mutation results
    ok = graphene.Boolean()

    class Arguments:
        id = graphene.Int()

    def mutate(root, info, id):
        workout = WorkoutModel.find_by_id(id)
        is_authorized(workout.author)
        # delete workout exercises associated with this workout
        WorkoutExerciseModel.deleteWorkoutExercises(id)

        # each workout should be unique to a program
        program_workout = ProgramWorkoutsModel.find_by_workout_id(id)
        program_workout.delete()

        workout.delete()
        ok = True

        return DeleteWorkout(ok=ok)

class Mutation(graphene.ObjectType):
    create_workout = CreateWorkout.Field()
    update_workout = UpdateWorkout.Field()
    delete_workout = DeleteWorkout.Field()


# !/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author: Marco
# @Date:   2016-04-17 14:40:30
# @Last Modified by:   Marco
# @Last Modified time: 2016-04-18 11:54:18

"""
Created on Sun Apr 17 15:17:17 2016

@author: Marco
"""

import argparse
# import scipy
import numpy
import pandas as pd

def parser_function():
    '''
    Function to parse arguments
    '''
    parser = argparse.ArgumentParser(description='''This program gets a
         text file with species names as input to create a taxonomic query on
         NCBI and retrieve taxonomic levels that have a defined Rank on NCBI
        ''')
    # Adding an argument to 'parse'
    parser.add_argument('in_file', type=str,
                        help='type the name and path of the input csv file')
    parser.add_argument('parameters', type=str,
                        help='path to txt file containing the parameters')
    parser.add_argument('out_file', type=str,
                        help='type the name and path of the output csv file')
    args = parser.parse_args()
    inp = args.in_file
    parameters = args.parameters
    outp = args.out_file
    return inp, parameters, outp

def text_to_list(txt_file):
    parameters = []
    with open(txt_file, 'r') as f:
        for line in f:
            parameters.append(line.strip("\n"))
    return parameters

def ci(df): return df.quantile(q=0.95)

def pivotation(amniotes, parameters):
    mean = pd.pivot_table(amniotes, values=parameters,
                          index=['class', 'order'], aggfunc=numpy.mean)
    mean.insert(0, column="Parameter",
                value="MEAN", allow_duplicates=True)
    minimum = pd.pivot_table(amniotes, values=parameters,
                             index=['class', 'order'], aggfunc=numpy.min)
    minimum.insert(0, column="Parameter",
                   value="MINIMUM", allow_duplicates=True)
    maximum = pd.pivot_table(amniotes, values=parameters,
                             index=['class', 'order'], aggfunc=numpy.max)
    maximum.insert(0, column="Parameter",
                   value="MAXIMUM", allow_duplicates=True)
    median = pd.pivot_table(amniotes, values=parameters,
                            index=['class', 'order'], aggfunc=numpy.median)
    median.insert(0, column="Parameter",
                  value="MEDIAN", allow_duplicates=True)
    ci95 = pd.pivot_table(amniotes, values=parameters,
                          index=['class', 'order'], aggfunc=ci)
    ci95.insert(0, column="Parameter",
                value="CI95", allow_duplicates=True)
    concate = mean.append([minimum, maximum, median, ci95])
    classes = []
    orders = []
    for x in concate.index:
        classes.append(x[0])
        orders.append(x[1])
    concate.insert(0, "Class", classes)
    concate.insert(1, "Order", orders)
    concate['index'] = list(range(len(concate)))
    concate = concate.set_index('index')
    return concate

def main():
    input_csv, parameters, output_csv = parser_function()
    amniotes = pd.read_csv(input_csv)
    amniotes = amniotes.replace(-999, numpy.nan)
    amniotes = amniotes.replace(float(-999), numpy.nan)
    amniotes = amniotes.replace(str(-999), numpy.nan)
    parameters = text_to_list(parameters)
    concate = pivotation(amniotes, parameters)
    concate.to_csv(output_csv)

if __name__ == '__main__':
    main()


import json

if __name__ == "__main__":

    questions_path = "C:/Users/benjavides/Desktop/Relation-Network-PyTorch/data/miniGQA/testing_question_ids.json"

    with open(questions_path, "r") as f:
        questions = json.load(f)

    yes=0
    no=0
    other=0

    for image_id in questions:
        question_dict = questions[image_id]
        answer = question_dict["answer"]
        if answer == "yes":
            yes+=1
        elif answer == "no":
            no+=1
        else:
            other+=1

    total = yes+no+other
    print(f"yes:{yes} no:{no} other:{other}")
    print(f"yes:{yes*100/total} no:{no*100/total} other:{other*100/total}")
#yes:41.828 no:56.558 other:28.898


"""Test FractionalBrownianMotion."""
# flake8: noqa
import pytest

from stochastic.continuous import FractionalBrownianMotion

def test_fractional_brownian_motion_str_repr(t, hurst):
    instance = FractionalBrownianMotion(t, hurst)
    assert isinstance(repr(instance), str)
    assert isinstance(str(instance), str)

def test_fractional_brownian_motion_sample(t, hurst, n, zero, threshold):
    instance = FractionalBrownianMotion(t, hurst)
    s = instance.sample(n, zero)
    assert len(s) == n + int(zero)


"""
    Tests for botogram/objects/messages.py

    Copyright (c) 2016 Pietro Albini <pietro@pietroalbini.io>
    Released under the MIT license
"""

import pytest

import botogram.objects.messages
import botogram.objects.chats

def get_dummy_message(text):
    """Get a dummy message with a custom text"""
    return botogram.objects.messages.Message({
        "message_id": 1,
        "from": {"id": 123, "first_name": "Nobody"},
        "chat": {"id": -123, "type": "chat", "title": "Something"},
        "date": 1,
        "text": text,
    })

def parsed_to_list(parsed):
    """Convert a ParsedText to a list of entities"""
    return [(e.type, e.text) for e in parsed]

###################
##  Parsed text  ##
###################

def test_parsed_text_entity():
    msg = get_dummy_message("Hey, I'm an http://url.com")

    # Try to create an entity without an attached message
    entity = botogram.objects.messages.ParsedTextEntity({
        "type": "url",
        "offset": 12,
        "length": 14,
    })
    assert entity.type == "link"
    # Those attributes requires a Message instance
    with pytest.raises(RuntimeError):
        entity.url
    with pytest.raises(RuntimeError):
        entity.text

    # Now attach a message
    entity.set_message(msg)
    assert entity.url == "http://url.com"
    assert entity.text == "http://url.com"

def test_parsed_text_entity_type():
    # Basic entity
    entity = botogram.objects.messages.ParsedTextEntity({
        "type": "",
        "offset": 0,
        "length": 23,
    })

    #################################
    #  A type which doesn't mutate  #
    #################################

    msg = get_dummy_message("#aaaaaaaaaaaaaaaaaaaaaaa")
    entity.set_message(msg)

    # Getter
    entity._type = "hashtag"
    assert entity.type == "hashtag"

    # Setter
    entity.type = "hashtag"
    assert entity._type == "hashtag"

    #########################
    #  A type which mutate  #
    #########################

    msg = get_dummy_message("/aaaaaaaaaaaaaaaaaaaaaaa")
    entity.set_message(msg)

    # Getter
    entity._type = "bot_command"
    assert entity.type == "command"

    # Setter
    entity.type = "command"
    assert entity._type == "bot_command"

    #####################################
    #  A link (which mutates as "url")  #
    #####################################

    msg = get_dummy_message("https://www.example.com")
    entity.set_message(msg)

    # Getter
    entity._type = "url"
    assert entity.type == "link"

    # Setter without URL explicitly defined
    entity.type = "link"
    assert entity._type == "url"

    # Setter with URL explicitly defined
    entity._url = "https://www.example.com"
    entity.type = "link"
    assert entity._type == "url"

    ###########################################
    #  A link (which mutates as "text_link")  #
    ###########################################

    msg = get_dummy_message("This is a labelled link")
    entity._url = "https://www.example.com"
    entity.set_message(msg)

    # Getter
    entity._type = "text_link"
    assert entity.type == "link"

    # Setter
    entity.type = "link"
    assert entity._type == "text_link"

    ############################################
    #  A mention (which mutates as "mention")  #
    ############################################

    msg = get_dummy_message("@test_image")
    entity.set_message(msg)

    # Getter
    entity._type = "mention"
    assert entity.type == "mention"

    # Setter
    entity.type = "mention"
    assert entity._type == "mention"

    #################################################
    #  A mention (which mutates as "text_mention")  #
    #################################################

    msg = get_dummy_message("@test_image")
    entity.user = botogram.objects.chats.User({"id": 1, "first_name": "Test"})
    entity.set_message(msg)

    # Getter
    entity._type = "text_mention"
    assert entity.type == "mention"

    # Setter
    entity.type = "mention"
    assert entity._type == "text_mention"

    entity.user = None

def test_parsed_text_entity_url():
    # Basic entity
    entity = botogram.objects.messages.ParsedTextEntity({
        "type": "",
        "offset": 0,
        "length": 23,
    })

    # Url entity
    msg = get_dummy_message("https://www.example.com")
    entity.type = "url"
    entity.set_message(msg)
    assert entity.url == "https://www.example.com"

    # Mention entity with @usernames
    msg = get_dummy_message("@a_botogram_users_group")
    entity.type = "mention"
    entity.set_message(msg)
    assert entity.url == "https://telegram.me/a_botogram_users_group"

    # Text mention entities without usernames
    msg = get_dummy_message("A botogram users group!")
    entity.user = botogram.objects.chats.User({"id": 1, "first_name": "Test"})
    entity.type = "mention"
    entity.set_message(msg)
    assert entity.url is None

    # Text mention entities with usernames
    entity.user.username = "test_username"
    assert entity.url == "https://telegram.me/test_username"

    entity.user = None

    # Email entity
    msg = get_dummy_message("john.42.doe@example.com")
    entity.type = "email"
    entity.set_message(msg)
    assert entity.url == "mailto:john.42.doe@example.com"

    # Non-url type with a forced url
    msg = get_dummy_message("this is not an url, obv")
    entity.type = "plain"
    entity._url = "https://www.example.com"
    entity.set_message(msg)
    assert entity.url == "https://www.example.com"
    entity._url = None

    # Url entity without a scheme
    msg = get_dummy_message("www.example.com/abcdefg")
    entity.type = "url"
    entity.set_message(msg)
    assert entity.url == "http://www.example.com/abcdefg"

def test_parsed_text():
    msg = get_dummy_message("This has an #hashtag!")
    parsed = botogram.objects.messages.ParsedText([
        {
            "type": "hashtag",
            "offset": 12,
            "length": 8,
        },
    ], message=msg)

    assert parsed_to_list(parsed) == [
        ("plain", "This has an "),
        ("hashtag", "#hashtag"),
        ("plain", "!"),
    ]
    assert parsed[0].text == "This has an "
    assert parsed[1].text == "#hashtag"
    assert parsed[2].text == "!"
    assert "plain" in parsed
    assert "hashtag" in parsed
    assert "url" not in parsed

def test_parsed_text_without_plain():
    # This needs an extra test because the "plain" types are not provided
    # directly by Telegram, but they're calculated dynamically by botogram
    msg = get_dummy_message("#hashtag")
    parsed = botogram.objects.messages.ParsedText([
        {
            "type": "hashtag",
            "offset": 0,
            "length": 8,
        },
    ], message=msg)

    assert parsed_to_list(parsed) == [
        ("hashtag", "#hashtag"),
    ]
    assert "plain" not in parsed

def test_parsed_text_filter():
    msg = get_dummy_message("I @mentioned #something!")
    parsed = botogram.objects.messages.ParsedText([
        {
            "type": "mention",
            "offset": 2,
            "length": 10,
        },
        {
            "type": "hashtag",
            "offset": 13,
            "length": 10,
        },
    ], message=msg)

    assert parsed_to_list(parsed) == [
        ("plain", "I "),
        ("mention", "@mentioned"),
        ("plain", " "),
        ("hashtag", "#something"),
        ("plain", "!"),
    ]
    assert parsed_to_list(parsed.filter("plain", "mention")) == [
        ("plain", "I "),
        ("mention", "@mentioned"),
        ("plain", " "),
        ("plain", "!"),
    ]
    assert parsed_to_list(parsed.filter("plain", exclude=True)) == [
        ("mention", "@mentioned"),
        ("hashtag", "#something"),
    ]


from .models import Server, Hero, Role, UserProfile
from rest_framework.serializers import ModelSerializer, ValidationError,SlugRelatedField
class ServerSerializer(ModelSerializer):
    class Meta:
        model = Server
        fields = '__all__'

class RoleSerializer(ModelSerializer):
    class Meta:
        model = Role
        fields = '__all__'

class HeroSerializer(ModelSerializer):
    class Meta:
        model = Hero
        fields = '__all__'

class UserProfileSerializer(ModelSerializer):
    fav_servers = SlugRelatedField(
        many=True,
        queryset=Server.objects.all(),
        slug_field='name',
    )

    fav_roles = SlugRelatedField(
        many=True,
        queryset=Role.objects.all(),
        slug_field='name',
    )

    fav_heroes = SlugRelatedField(
        many=True,
        queryset=Hero.objects.all(),
        slug_field='name',
    )
    def validate_fav_servers(self, value):
        if len(value) > 3:
            raise ValidationError("Too many servers selected, maximum is 3")
        return value

    def validate_fav_heroes(self, value):
        if len(value) > 5:
            raise ValidationError("Too many heroes selected, maximum is 5")
        return value

    def validate_fav_roles(self, value):
        if len(value) > 3:
            raise ValidationError("Too many roles selected, maximum is 3")
        return value
    # check if the user is logged in
    def validate_user(self, value):
        if self.context.get('request'):
              if self.context['request'].user != value:
                raise ValidationError("You do not have the permission to create or this resource")
        else:
            raise ValidationError("Login?")
        return value
    class Meta:
        model = UserProfile
        fields = ('fav_servers', 'fav_heroes', 'id', 'fav_roles')


import os
from inspect import getmembers
from jinja2 import Markup
from flask import Flask, jsonify, render_template
from flask_jwt_extended import JWTManager
from flask_cache import Cache
from application import config

TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__))
if not config.USE_REDIS:
    cache = Cache(config={'CACHE_TYPE': 'simple'})
if not config.USE_CACHE:
    cache = Cache(config={'CACHE_TYPE': 'null'})
else:
    cache = Cache(config=config.REDIS_CONFIG)

# Bundle app creation in a function for use with testing and easier config
def create_app(app_config=None):
    global cache
    if app_config:
        for member in getmembers(app_config):
            setattr(config, member[0], member[1])
    # Initialize application, DB, and cache
    app = Flask(__name__)
    app.config.from_object(config)

    with app.app_context():
        # Initialize DB
        from application.base_models import db
        db.init_app(app)
        if config.USE_GEVENT:
            db.engine.pool._use_threadlocal = True
            db.engine.pool.use_threadlocal = True
        # register the cache to the app
        cache.init_app(app)

        from application.exceptions import InvalidAPIRequest, get_traceback
        from application.auth.auth_utilities import JWTUtilities
        # Initialize JWT handler
        jwt = JWTManager(app)

        @jwt.user_claims_loader
        def add_use_claims(user):
            return JWTUtilities.add_user_claims(user)

        @jwt.user_identity_loader
        def get_user_identity(user):
            return JWTUtilities.get_user_identity(user)

        # Initializing Routes/Modules
        from application.controllers import home_blueprint
        from application.auth.controllers import auth_blueprint
        from application.recipe.controllers import recipe_blueprint
        from application.user.controllers import user_blueprint
        from application.admin.controllers import admin_blueprint
        blueprints = [
            home_blueprint,
            auth_blueprint,
            recipe_blueprint,
            user_blueprint,
            admin_blueprint
        ]

        for blueprint in blueprints:
            setattr(blueprint, 'template_folder',
                    os.path.join(
                        TEMPLATE_DIR,
                        blueprint.template_folder
                    ))
            app.register_blueprint(blueprint)

        # for serving static admin pages
        app.jinja_env.globals['include_raw'] = lambda filename: Markup(app.jinja_loader.get_source(app.jinja_env, filename)[0])

        # Initialize general API Error handler function
        @app.errorhandler(Exception)
        def handle_api_error(error):
            db.session.remove()
            if isinstance(error, InvalidAPIRequest):
                if error.status_code != 404:
                    if error.status_code == 500 and config.DEBUG:
                        print(get_traceback())
                    response = jsonify(error.to_dict())
                    response.status_code = error.status_code
                    return response
                return render_template('404.html')
            else:
                app.logger.error("Unknown error: {0}. Stacktrace: {1}".format(str(error), get_traceback()))

        # Teardown to close/dremove db sessions after requests
        @app.teardown_appcontext
        def remove_session(exception=None):
            db.session.remove()

        return app


# k_means.py
# Implementation of the K-Means algorithm for finding clusters.
import random as rand
import src.util as util

# The smallest percentage change in distortion that will NOT stop the centroid calculation.
DISTORTION_CHANGE_CUTOFF = 0.001

# The K-Means algorithm implementation. This will find the k clusters in the training data and then store the class
# distribution of the clusters to use for a nearest neighbor search done by 'run'.
class KMeans:

    # Creates an instance of the algorithm and instantly begins clustering. This way, the run method can be used as
    # soon as the object is created.
    def __init__(self, training_data, k):
        self.training_data = training_data.copy()
        self.k = k
        # The centroids and cluster_classes list are used to represent the clusters themselves. centroids[i] is the
        # centroid for the ith cluster, and cluster_classes[i] is a map of each class to the probability it occurs in
        # the ith cluster. Note that any classes with 0% probabilities (no training data in the cluster had that class)
        # are not included in the dictionary. Lastly, we also store the distortion of the final clusters.
        self.centroids, self.clusters, self.cluster_classes, self.distortion = self.calculate_clusters()

    # Returns the centroids representing each cluster and the  distribution of classes within each cluster. Both values
    # are lists, where index i in either list represents the ith cluster. The class distributions are in the form of a
    # dictionary that maps each class to the frequency of the class in the cluster.
    def calculate_clusters(self):
        # First, generate centroids randomly
        centroids = self.generate_random_centroids()
        clusters = []

        distortion = None
        distortion_change = None
        # The cluster calculation will continue until we have "converged", which is measured by having a smaller
        # percentage change in distortion between two cycles than DISTORTION_CHANGE_CUTOFF.
        while distortion_change is None or abs(distortion_change) >= DISTORTION_CHANGE_CUTOFF:
            # Reset the clusters to have 0 observations.
            clusters = [[] for i in range(self.k)]

            # Assign each observation to the cluster corresponding to the closest centroid.
            for obs in self.training_data.get_data():
                closest_centroid_i = self.find_closest_centroid(obs, centroids)
                clusters[closest_centroid_i].append(obs)

            # With all observations assigned to a cluster, we now recalculate the centroids as the cluster means.
            for i in range(self.k):
                if len(clusters[i]) != 0:
                    centroids[i] = self.calculate_cluster_mean(clusters[i])

            # Update distortion variables to determine whether to exit the loop now.
            new_distortion = self.calculate_distortion(clusters, centroids)
            if distortion is not None:
                # Percentage change in distortion from the previous set of clusters to these new clusters:
                distortion_change = (new_distortion - distortion) / distortion
            distortion = new_distortion

        # Finally, instead of returning each cluster itself, we are only interested in knowing the class distributions
        # in each cluster.
        cluster_classes = [{} for i in range(len(clusters))]
        for i in range(len(clusters)):
            cluster_classes[i] = util.calculate_class_distribution(clusters[i], self.training_data.class_col)

        return centroids, clusters, cluster_classes, distortion

    # Creates a list of centroids that are "random." Returns the centroids as a 2D list.
    def generate_random_centroids(self):
        centroids = [[] for i in range(self.k)]
        # Initialize the means using randomly selected attribute values.
        for i in range(self.k):
            centroids[i] = self.generate_random_centroid()
        return centroids

    # Creates a single random centroid. A centroid is really just a vector of attribute values, so we accomplish this
    # by randomly selecting an attribute value from our training data.
    def generate_random_centroid(self):
        num_cols = len(self.training_data.get_data()[0])
        num_training_data = len(self.training_data.get_data())

        # Initialize the centroid with None values for all columns
        centroid = [None for i in range(num_cols)]
        # For the attribute columns, assign a value picked randomly from the training set.
        for attr_col in self.training_data.attr_cols:
            # Random pick from the training set:
            selected_obs = self.training_data.get_data()[rand.randrange(0, num_training_data)]
            # Assigning that pick's value for the current attribute:
            centroid[attr_col] = selected_obs[attr_col]
        return centroid

    # Out of the list of all centroids, this will return the corresponding index for the centroid that is closest to the
    # given observation.
    def find_closest_centroid(self, obs, centroids):
        # Index of the closest centroid seen so far.
        closest_centroid_i = None
        # Distance to the closest centroid seen so far.
        min_dist = None
        for i in range(len(centroids)):
            dist = self.training_data.distance(centroids[i], obs)
            if closest_centroid_i is None or dist < min_dist:
                closest_centroid_i = i
                min_dist = dist
        return closest_centroid_i

    # Calculates the mean of the cluster. This is tricky because not all attributes are numeric -- some are strings.
    # When we encounter a string attribute, we simply calculate the mode (using a frequency dictionary) and use that
    # as the "mean".
    def calculate_cluster_mean(self, cluster):
        num_cols = len(self.training_data.get_data()[0])
        attr_cols = self.training_data.attr_cols

        # If the cluster has to items in it, we return the mean as a random coordinate.
        if len(cluster) == 0:
            return self.generate_random_centroid()

        # The sums and for calculating the average and mode of the attributes.
        sums = [0 for i in range(num_cols)]

        # This section calculates the sum of each observation's attribute.
        for obs in cluster:
            # We iterate over the attribute columns only (to ignore class and unused columns)
            for attr_col in attr_cols:
                sums[attr_col] += obs[attr_col]

        # We now calculate the "means", which is an average for numeric columns and the mode for string columns.
        mean = [0 for i in range(num_cols)]
        # We again want to just iterate over the attribute columns instead of all the columns
        for attr_col in attr_cols:
                mean[attr_col] = sums[attr_col] / len(cluster)

        # Lastly, we return this mean.
        return mean

    # Calculates the distortion of the given clusters using the training data. Essentially, distortion is a measure of
    # how far each point is from the cluster it belongs to. We use this to optimize our centroid placement.
    def calculate_distortion(self, clusters, centroids):
        distortion = 0
        for cluster_i in range(len(clusters)):
            for obs in clusters[cluster_i]:
                centroid = centroids[cluster_i]
                distortion += self.training_data.distance(obs, centroid)
        return distortion

    # Classifies an example by finding the nearest cluster and then returning the class distribution for that cluster.
    def run(self, example):
        closest_centroid_i = self.find_closest_centroid(example, self.centroids)
        return self.cluster_classes[closest_centroid_i]


from operator import attrgetter

import pytest

from regionmask import Regions, defined_regions
from regionmask.defined_regions import _maybe_get_column

from . import has_cartopy, requires_cartopy
from .utils import (
    REGIONS,
    REGIONS_DEPRECATED,
    REGIONS_REQUIRING_CARTOPY,
    get_naturalearth_region_or_skip,
)

def _test_region(region, n_regions):
    assert isinstance(region, Regions)
    assert len(region) == n_regions

    # currently all regions are -180..180
    assert region.lon_180

@pytest.mark.parametrize("region_name, n_regions", REGIONS.items())
def test_defined_region(region_name, n_regions):

    region = attrgetter(region_name)(defined_regions)

    _test_region(region, n_regions)

@pytest.mark.parametrize("region_name, n_regions", REGIONS_DEPRECATED.items())
def test_defined_region_deprecated(region_name, n_regions):

    region = attrgetter(region_name)(defined_regions)

    _test_region(region, n_regions)

@requires_cartopy
@pytest.mark.parametrize("region_name, n_regions", REGIONS_REQUIRING_CARTOPY.items())
def test_defined_regions_natural_earth(monkeypatch, region_name, n_regions):

    region = get_naturalearth_region_or_skip(monkeypatch, region_name)

    _test_region(region, n_regions)

@requires_cartopy
def test_natural_earth_loaded_as_utf8():
    # GH 95
    regions = defined_regions.natural_earth.ocean_basins_50
    r = regions[90]

    assert r.name == "Río de la Plata"

def test_maybe_get_column():
    class lowercase:
        @property
        def name(self):
            return 1

    class uppercase:
        @property
        def NAME(self):
            return 2

    assert _maybe_get_column(lowercase(), "name") == 1
    assert _maybe_get_column(uppercase(), "name") == 2
    assert _maybe_get_column(uppercase(), "NAME") == 2

    with pytest.raises(KeyError, match="not on the geopandas dataframe"):
        _maybe_get_column(lowercase, "not_a_column")

@pytest.mark.skipif(has_cartopy, reason="should run if cartopy is _not_ installed")
def test_natural_earth_raises_without_cartopy():

    with pytest.raises(ImportError, match="cartopy is required"):
        defined_regions.natural_earth.countries_110


import unidecode
import string
import random
import re

all_characters = string.printable
n_characters = len(all_characters)
limit = 1000000
f = open('speeches.txt', encoding='latin-1').read()
file = unidecode.unidecode(f)
file_len = len(file)
print('file_len =', file_len)

f_s = open('shakespeare_input.txt', encoding='latin-1').read()
file_s = unidecode.unidecode(f_s)
file_len_s = len(file_s)
print('file_len_s =', file_len_s)

# To make inputs out of this big string of data, we will be splitting it into chunks.

chunk_len = 1000

def random_chunk():
    start_index = random.randint(0, file_len - chunk_len)
    end_index = start_index + chunk_len + 1
    return file[start_index:end_index]

def random_chunk_s():
    start_index = random.randint(0, file_len_s - chunk_len)
    end_index = start_index + chunk_len + 1
    return file_s[start_index:end_index]

print(random_chunk())
print(random_chunk_s())

# # Build the Model

import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F

class RNN(nn.Module):
    def __init__(self, input_size=n_characters, hidden_size=128, output_size=n_characters, n_layers=2):
        super(RNN, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.n_layers = n_layers

        self.encoder = nn.Embedding(input_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size, n_layers)
        self.decoder = nn.Linear(hidden_size, output_size)

    def forward(self, input, hidden):
        input = self.encoder(input.view(1, -1))
        output, hidden = self.gru(input.view(1, 1, -1), hidden)
        output = self.decoder(output.view(1, -1))
        #output = F.log_softmax(output, dim=1)
        return output, hidden

    def init_hidden(self):
        return Variable(torch.zeros(self.n_layers, 1, self.hidden_size))

# # Inputs and Targets

# Turn string into list of longs
def char_tensor(string):
    tensor = torch.zeros(len(string)).long()
    for c in range(len(string)):
        tensor[c] = all_characters.index(string[c])
    return Variable(tensor)

print(char_tensor('abcDEF'))

def random_training_set():
    chunk = random_chunk()
    inp = char_tensor(chunk[:-1])
    target = char_tensor(chunk[1:])
    return inp, target

# # Evaluating

def category_from_output(output):
    top_n, top_i = output.data.topk(1) # Tensor out of Variable with .data
    category_i = top_i[0][0]
    return all_characters[category_i], category_i

def evaluate(prime_str='A', predict_len=100, temperature=0.8):
    hidden = decoder.init_hidden()
    prime_input = char_tensor(prime_str)
    predicted = prime_str
    perplexity = 1

    # Use priming string to "build up" hidden state
    for p in range(len(prime_str) - 1):
        _, hidden = decoder(prime_input[p], hidden)
    inp = prime_input[-1]

    for p in range(predict_len):
        output, hidden = decoder(inp, hidden)

        #char_i,top_i = category_from_output(output)
        #print(char_i,top_i)
        #Pick the one with the highest category

        #Sample from the network as a multinomial distribution
        output_dist = output.data.view(-1).div(temperature).exp()
        top_i = torch.multinomial(output_dist, 1)[0]

        # Add predicted character to string and use as next input
        predicted_char = all_characters[top_i]
        predicted += predicted_char
        inp = char_tensor(predicted_char)

    return predicted

def perplexity(input_string):
    hidden = decoder.init_hidden()
    perplexity = 0
    n = len(input_string)

    inp = char_tensor(input_string[0])

    for p in range(n):
        output, hidden = decoder(inp, hidden)

        inp = char_tensor(input_string[p])

        output = F.softmax(output, dim=1)

        output_dist = output.data.view(-1).tolist()

        perplexity += math.log10 ( output_dist [ all_characters.index(input_string[p]) ] )
    perplexity *= -1/n

    return 10**perplexity

# # Training

# A helper to print the amount of time passed:

import time, math

def time_since(since):
    s = time.time() - since
    m = math.floor(s / 60)
    s -= m * 60
    return '%dm %ds' % (m, s)

# The main training function

def train(inp, target):
    hidden = decoder.init_hidden()
    decoder.zero_grad()
    loss = 0

    for c in range(chunk_len):
        output, hidden = decoder(inp[c], hidden)
        loss += criterion(output, target[c].unsqueeze(0))

    loss.backward()
    decoder_optimizer.step()

    return loss.item() / chunk_len

# Then we define the training parameters, instantiate the model, and start training:

n_epochs = 7000
print_every = 100
plot_every = 100
hidden_size = 100
n_layers = 2
lr = 0.005

decoder = RNN(n_characters, hidden_size, n_characters, n_layers)
decoder_optimizer = torch.optim.Adam(decoder.parameters(), lr=lr)
criterion = nn.CrossEntropyLoss()

start = time.time()
all_losses = []
loss_avg = 0

shakespeare_perplexity = []
corpus_preplexity = []

corpus_random_chunk = random_chunk()
shakespeare_random_chunk = random_chunk_s()

#print( ' Trump Speeches Random Text: ',corpus_random_chunk)
#print( ' Shakespeare Random Text: ',shakespeare_random_chunk)
for epoch in range(1, n_epochs + 1):

    loss = train(*random_training_set())
    loss_avg += loss

    if epoch % print_every == 0:
        print('[%s (%d %d%%) %.4f]' % (time_since(start), epoch, epoch / n_epochs * 100, loss))
        print(evaluate('Wh', 100), '\n')
        shakespeare_perplexity.append(perplexity(random_chunk_s()))
        corpus_preplexity.append (perplexity(random_chunk()))

    if epoch % plot_every == 0:
        all_losses.append(loss_avg / plot_every)
        loss_avg = 0

#Save the PyTorch Model

models_generate = decoder
torch.save(models_generate.state_dict(), "/Users/srinivassuri/Documents/cis530/hw6/part3_submission/model_generate")

def evaluate_model(model,prime_str='A', predict_len=100, temperature=0.8):
    hidden = model.init_hidden()
    prime_input = char_tensor(prime_str)
    predicted = prime_str
    perplexity = 1

    # Use priming string to "build up" hidden state
    for p in range(len(prime_str) - 1):
        _, hidden = model(prime_input[p], hidden)
    inp = prime_input[-1]

    for p in range(predict_len):
        output, hidden = model(inp, hidden)

        #Sample from the network as a multinomial distribution
        output_dist = output.data.view(-1).div(temperature).exp()
        top_i = torch.multinomial(output_dist, 1)[0]

        # Add predicted character to string and use as next input
        predicted_char = all_characters[top_i]
        predicted += predicted_char
        inp = char_tensor(predicted_char)

    return predicted

#Load the PyTorch Model back and generate few sentences
model = RNN(n_characters, hidden_size, n_characters, n_layers)
model.load_state_dict(torch.load("/Users/srinivassuri/Documents/cis530/hw6/part3_submission/model_generate"))
model.eval() #To predict

print( evaluate_model(model," ",200,0.5 ) )

#Text generated by the model
print(evaluate('Wh', 100), '\n')
print(evaluate('Wh', 100), '\n')
print(evaluate('Wh', 100), '\n')

#Code from the previous assignment for N-Grams
def start_pad(n):
    ''' Returns a padding string of length n to append to the front of text
        as a pre-processing step to building n-grams '''
    return '~' * n

def ngrams(n, text):
    ''' Returns the ngrams of the text as tuples where the first element is
        the length-n context and the second is the character '''
    m = len(text)

    #print(type(text),len(text),text)

    ngrams_ = []

    if( n == 0 ):
        for i in range(m):
            ngrams_.append( ('',text[i]) )
        return ngrams_

    for i in range(m):
        res = ''
        for j in range(i-n,i,1): #[i-n,i-1]
            if( j < 0 ):
                res += '~'
            else:
                res += text[j]
        ngrams_.append( (res,text[i]) )

    return ngrams_

class NgramModel(object):
    ''' A basic n-gram model using add-k smoothing '''

    def __init__(self, n, k):
        self.vocab = set()
        self.n = n
        self.k = k
        self.counts = {}
        self.contexts ={} #Count of the Contexts
        self.contexts_tokens={} #The actual tokens that occur after the context

    def get_vocab(self):
        ''' Returns the set of characters in the vocab '''
        return self.vocab

    def update(self, text):
        ''' Updates the model n-grams based on text '''
        #print('Updating...',text)
        #Add the Characters to the set if they don't exist
        for ch in text:
            #print('ch is',ch)
            if ch not in self.vocab:
                self.vocab.add(ch)

        #Update the Counts of the ngrams
        ngrams_ = ngrams(self.n, text)

        for (context,char) in ngrams_:
            if (context,char) not in self.counts:
                self.counts[(context,char)] = 1
            else:
                self.counts[(context,char)] += 1

            if context not in self.contexts:
                self.contexts[context] = 1
                self.contexts_tokens[context] = [char]
            else:
                self.contexts[context] += 1
                if char not in self.contexts_tokens[context]:
                    words = self.contexts_tokens[context]
                    words.append(char)
                    self.contexts_tokens[context] = words

    def prob(self, context, char):
        ''' Returns the probability of char appearing after context '''
        # P( char | context ) = P( context, char ) / P( context )
        if context not in self.contexts:
            #print('context not seen',context)
            return 1/(len(self.get_vocab() ))

        if( self.k == 0 ):
            if (context,char) not in self.counts:
                return 0.0

            return self.counts[ (context,char) ] / self.contexts[context]
        else:
            #K Smoothing
            if (context,char) not in self.counts:
                return (self.k)/(self.contexts[context]+(self.k*len(self.get_vocab()) ))

            return (self.counts[ (context,char) ] + self.k) / (self.contexts[context] + (self.k*len(self.get_vocab())) )

    def random_char(self, context):
        ''' Returns a random character based on the given context and the
            n-grams learned by this model '''

        words = sorted(self.get_vocab()) #Sorted Lexicographically

        r = random.random()
        #print(r)
        res = 0

        for word in words:
            #print(word,self.prob( context,word ) )
            if res + self.prob( context,word ) > r:
                return word
            else:
                res += self.prob( context,word )

    def random_text(self, length):
        ''' Returns text of the specified character length based on the
            n-grams learned by this model '''
        #print('Generate text for length',length,self.n,self.k)
        context = start_pad(self.n)

        text = ''

        for i in range(length):
            random_char = self.random_char(context)
            context = context[1:]
            context += random_char
            text += random_char

        return text

    def perplexity(self, text):
        ''' Returns the perplexity of text based on the n-grams learned by
            this model '''
        ngrams_ = ngrams(self.n,text)

        res = 0

        for (context,char) in ngrams_:
            prob = self.prob( context,char )
            if( prob == 0 ):
                return float('inf')
            else:
                res += math.log10( self.prob( context,char ))
        res *= -1/len(text)

        return 10**res

class NgramModelWithInterpolation(NgramModel):
    ''' An n-gram model with interpolation '''

    def __init__(self, n, k):
        self.vocab = set()
        self.n = n
        self.k = k
        self.counts = {}
        self.contexts ={} #Count of the Contexts
        self.contexts_tokens={} #The actual tokens that occur after the context

        self.w = []
        for i in range(self.n + 1): #[n+1] weights
             self.w.append(  1.0/(n+1) )

    def get_vocab(self):
        return self.vocab

    def update(self, text):
        for ch in text:
            if ch not in self.vocab:
                self.vocab.add(ch)

        #Update the Counts of the ngrams
        n = self.n
        while n >=0 :
            ngrams_ = ngrams(n, text)

            for (context,char) in ngrams_:
                if (context,char) not in self.counts:
                    self.counts[(context,char)] = 1
                else:
                    self.counts[(context,char)] += 1

                if context not in self.contexts:
                    self.contexts[context] = 1
                    self.contexts_tokens[context] = [char]
                else:
                    self.contexts[context] += 1
                    if char not in self.contexts_tokens[context]:
                        words = self.contexts_tokens[context]
                        words.append(char)
                        self.contexts_tokens[context] = words
            n = n-1

    def prob(self, context, char):
        res = 0
        for i in range(self.n + 1 ):

            if( i != 0 ):
                context = context[1:]

            if context not in self.contexts:
                res += ( self.w[i]*1/(len(self.get_vocab() )) )
                continue

            if( self.k == 0 ):
                if (context,char) not in self.counts:
                    res += 0.0
                    continue

                res +=  self.w[i] * (self.counts[ (context,char) ] / self.contexts[context] )
            else:
                #K Smoothing
                if (context,char) not in self.counts:
                    res += self.w[i] * (self.k)/(self.contexts[context]+(self.k*len(self.get_vocab()) ))
                    continue

                res += self.w[i] * (self.counts[ (context,char) ] + self.k) / (self.contexts[context] + (self.k*len(self.get_vocab())) )

        return res

def perplexity_model(model,input_string):
    hidden = model.init_hidden()
    perplexity = 0
    n = len(input_string)

    inp = char_tensor(input_string[0])

    for p in range(1,n,1):
        output, hidden = model(inp, hidden)

        inp = char_tensor(input_string[p])

        output = F.softmax(output, dim=1)

        output_dist = output.data.view(-1).tolist()

        perplexity += math.log10 ( output_dist [ all_characters.index(input_string[p]) ] )
    perplexity *= -1/n

    return 10**perplexity

#Create a New N-gramModel
ng = NgramModelWithInterpolation(3,2)

f1_s = open('speeches.txt', encoding='latin-1')
i = 0

for line in f1_s.readlines():
    ng.update(line)
    i +=1

# In[322]:

print('S.no\tngram perplexity\trnn perplexity')
ng_ = []
rnn_ = []

for i in range(20):
    chunk = random_chunk()
    #print(' Chunk is: ', chunk)
    ng_.append(ng.perplexity(chunk))
    rnn_.append(perplexity_model(model,chunk ))
    print(str(i)+'\t'+str(ng_[i])+'\t'+ str(rnn_[i]) )
print('------------------------------------------------------')
print('avg:\t' ,sum(ng_)/20, '\t' , sum(rnn_)/20 )


n=int(input())
s=input().split()
acount=0
pcount=0
for i in range(n):
    if s[i]=="A":
        acount=acount+1
    elif s[i]=="P":
        pcount=pcount+1
if((pcount/n)>0.25):
    print("Not Blacklisted")
else:
    print("Blacklisted")


"""
Course model serializers
"""
from urllib.parse import urljoin

from django.conf import settings
from django.templatetags.static import static
from rest_framework import serializers

from courses import models

def _get_thumbnail_url(page):
    """
    Get the thumbnail URL or else return a default image URL.

    Args:
        page (cms.models.ProductPage): A product page

    Returns:
        str:
            A page URL
    """
    relative_url = (
        page.thumbnail_image.file.url
        if page
        and page.thumbnail_image
        and page.thumbnail_image.file
        and page.thumbnail_image.file.url
        else static("images/mit-dome.png")
    )
    return urljoin(settings.SITE_BASE_URL, relative_url)

class BaseCourseSerializer(serializers.ModelSerializer):
    """Basic course model serializer"""

    class Meta:
        model = models.Course
        fields = ["id", "title", "readable_id"]

class BaseCourseRunSerializer(serializers.ModelSerializer):
    """Minimal CourseRun model serializer"""

    class Meta:
        model = models.CourseRun
        fields = [
            "title",
            "start_date",
            "end_date",
            "enrollment_start",
            "enrollment_end",
            "expiration_date",
            "courseware_url",
            "courseware_id",
            "run_tag",
            "id",
        ]

class CourseRunSerializer(BaseCourseRunSerializer):
    """CourseRun model serializer"""

    class Meta:
        model = models.CourseRun
        fields = BaseCourseRunSerializer.Meta.fields

class CourseSerializer(serializers.ModelSerializer):
    """Course model serializer - also serializes child course runs"""

    courseruns = serializers.SerializerMethodField()
    next_run_id = serializers.SerializerMethodField()
    topics = serializers.SerializerMethodField()

    def get_next_run_id(self, instance):
        """Get next run id"""
        run = instance.first_unexpired_run
        return run.id if run is not None else None

    def get_courseruns(self, instance):
        """Unexpired and unenrolled course runs"""
        all_runs = self.context.get("all_runs", False)
        if all_runs:
            active_runs = instance.unexpired_runs
        else:
            user = self.context["request"].user if "request" in self.context else None
            active_runs = (
                instance.available_runs(user)
                if user and user.is_authenticated
                else instance.unexpired_runs
            )
        return [
            CourseRunSerializer(instance=run, context=self.context).data
            for run in active_runs
            if run.live
        ]

    def get_topics(self, instance):
        """List topics of a course"""
        return sorted(
            [{"name": topic.name} for topic in instance.topics.all()],
            key=lambda topic: topic["name"],
        )

    class Meta:
        model = models.Course
        fields = [
            "id",
            "title",
            "readable_id",
            "courseruns",
            "next_run_id",
            "topics",
        ]

class CourseRunDetailSerializer(serializers.ModelSerializer):
    """CourseRun model serializer - also serializes the parent Course"""

    course = BaseCourseSerializer(read_only=True)

    class Meta:
        model = models.CourseRun
        fields = [
            "course",
            "title",
            "start_date",
            "end_date",
            "enrollment_start",
            "enrollment_end",
            "expiration_date",
            "courseware_url",
            "courseware_id",
            "id",
        ]

class BaseProgramSerializer(serializers.ModelSerializer):
    """Basic program model serializer"""

    class Meta:
        model = models.Program
        fields = ["title", "readable_id", "id"]

class ProgramSerializer(serializers.ModelSerializer):
    """Program model serializer"""

    courses = serializers.SerializerMethodField()
    start_date = serializers.SerializerMethodField()
    end_date = serializers.SerializerMethodField()
    enrollment_start = serializers.SerializerMethodField()
    topics = serializers.SerializerMethodField()

    def get_courses(self, instance):
        """Serializer for courses"""
        return CourseSerializer(
            instance.courses.filter(live=True).order_by("position_in_program"),
            many=True,
        ).data

    def get_start_date(self, instance):
        """
        start_date is the starting date for the earliest live course run for all courses in a program

        Returns:
            datetime: The starting date
        """
        return (
            models.CourseRun.objects.filter(course__program=instance, live=True)
            .order_by("start_date")
            .values_list("start_date", flat=True)
            .first()
        )

    def get_end_date(self, instance):
        """
        end_date is the end date for the latest live course run for all courses in a program.

        Returns:
            datetime: The ending date
        """
        return (
            models.CourseRun.objects.filter(course__program=instance, live=True)
            .order_by("end_date")
            .values_list("end_date", flat=True)
            .last()
        )

    def get_enrollment_start(self, instance):
        """
        enrollment_start is first date where enrollment starts for any live course run
        """
        return (
            models.CourseRun.objects.filter(course__program=instance, live=True)
            .order_by("enrollment_start")
            .values_list("enrollment_start", flat=True)
            .first()
        )

    def get_topics(self, instance):
        """List all topics in all courses in the program"""
        topics = (
            models.CourseTopic.objects.filter(course__program=instance)
            .values("name")
            .distinct("name")
        )
        return list(topics)

    class Meta:
        model = models.Program
        fields = [
            "title",
            "readable_id",
            "id",
            "courses",
            "start_date",
            "end_date",
            "enrollment_start",
            "topics",
        ]

class CourseRunEnrollmentSerializer(serializers.ModelSerializer):
    """CourseRunEnrollment model serializer"""

    run = CourseRunDetailSerializer(read_only=True)

    class Meta:
        model = models.CourseRunEnrollment
        fields = ["run"]

class ProgramEnrollmentSerializer(serializers.ModelSerializer):
    """ProgramEnrollmentSerializer model serializer"""

    program = BaseProgramSerializer(read_only=True)
    course_run_enrollments = serializers.SerializerMethodField()

    def __init__(self, *args, **kwargs):
        assert (
            "context" in kwargs and "course_run_enrollments" in kwargs["context"]
        ), "An iterable of course run enrollments must be passed in the context (key: course_run_enrollments)"
        super().__init__(*args, **kwargs)

    def get_course_run_enrollments(self, instance):
        """Returns a serialized list of course run enrollments that belong to this program (in position order)"""
        return CourseRunEnrollmentSerializer(
            sorted(
                (
                    enrollment
                    for enrollment in self.context["course_run_enrollments"]
                    if enrollment.run.course.program_id == instance.program.id
                ),
                key=lambda enrollment: enrollment.run.course.position_in_program,
            ),
            many=True,
        ).data

    class Meta:
        model = models.ProgramEnrollment
        fields = [
            "id",
            "program",
            "course_run_enrollments",
        ]


import logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

import collections
import json
import os

import numpy as np

class Vocab(object):
    '''Holding vocabulary.

    Core info:
        words: a list of words
        freqs: a list of frequencies
        unk_size: size of unknown words
        sample_factor: factor for (sub) sampling by rising frequency to this power.

    Infered:
        anything else :-D
    '''

    def __init__(self, counter=None, known_size=None, unk_size=1, sample_factor=3./4.):
        if counter is None:
            return
        if known_size is None:
            known_size = len(counter)
        assert known_size <= len(counter)

        most_common = counter.most_common(known_size)
        words, freqs = zip(*most_common)

        core = self.core = {
            'words': words,
            'freqs': freqs,
            'unk_size': unk_size,
            'sample_factor': sample_factor,
        }

        self._infer()

    def _infer(self):
        words = self.words = self.core['words']
        freqs = self.freqs = self.core['freqs']
        unk_size = self.unk_size = self.core['unk_size']
        sample_factor = self.sample_factor = self.core['sample_factor']

        known_size = self.known_size = len(words)
        vocab_size = self.vocab_size = known_size + unk_size
        logger.info('infer vocab with known_size = %d and unk_size = %d' % (
                self.known_size, self.unk_size))

        self.idx2str = self.words
        self.str2idx = {v:i for (i, v) in enumerate(self.idx2str)}

        weight = np.array(self.freqs, dtype=np.float32)
        weight = weight ** sample_factor
        weight = weight / weight.sum()
        weight_prefix_sum = [0.] * len(weight)
        weight_prefix_sum[0] = weight[0]
        for i in range(1, len(weight)):
            weight_prefix_sum[i] = weight[i] + weight_prefix_sum[i-1]
        self.weight = np.array(weight)
        self.weight_prefix_sum = np.array(weight_prefix_sum)

    def save(self, fp):
        if isinstance(fp, str):
            fp = open(fp, 'w')
        json.dump(self.core, fp)

    def load(self, fp):
        if isinstance(fp, str):
            fp = open(fp, 'r')
        self.core = json.load(fp)
        self._infer()

    def encode(self, obj):
        if isinstance(obj, list):
            return [self.encode(_) for _ in obj]
        elif isinstance(obj, str):
            index = self.str2idx.get(obj, -1)
            if index == -1:
                index = self.known_size + hash(obj) % self.unk_size
            return index
        else:
            assert False

    def decode(self, obj):
        if isinstance(obj, list):
            return [self.decode(_) for _ in obj]
        elif isinstance(obj, int):
            assert 0 <= obj < self.vocab_size
            if obj < self.known_size:
                return self.idx2str[obj]
            else:
                return 'unk_%d' % (obj - self.known_size)
        else:
            assert False

    def sample_single(self):
        p = np.random.rand()
        idx = np.searchsorted(self.weight_prefix_sum, p)
        idx = min(idx, self.known_size - 1)
        return idx

    def sample(self, num):
        return [self.sample_single() for _ in range(num)]

def test_vocab():
    import tempfile

    counter = collections.Counter(['a', 'a', 'b', 'b', 'b', 'c'])
    vocab = Vocab(counter, known_size=2, unk_size = 2)
    assert vocab.known_size == 2
    assert vocab.unk_size == 2
    assert vocab.vocab_size == 4
    assert vocab.encode('b') == 0
    assert vocab.encode('a') == 1
    assert vocab.encode(['a', 'b']) == [1, 0]
    assert vocab.decode(0) == 'b'
    assert vocab.decode(1) == 'a'
    assert vocab.decode([0, 1, 2, 3]) == ['b', 'a', 'unk_0', 'unk_1']

    sampled = vocab.sample(100)


import warnings
from collections import MutableMapping

import numpy as np
import pandas as pd

from autocnet.utils import utils
from autocnet.matcher import health
from autocnet.matcher import outlier_detector as od
from autocnet.matcher import suppression_funcs as spf
from autocnet.matcher import subpixel as sp
from autocnet.matcher.feature import FlannMatcher
from autocnet.transformation.transformations import FundamentalMatrix, Homography
from autocnet.vis.graph_view import plot_edge
from autocnet.cg import cg

class Edge(dict, MutableMapping):
    """
    Attributes
    ----------
    source : hashable
             The source node

    destination : hashable
                  The destination node
    masks : set
            A list of the available masking arrays

    provenance : dict
                 With key equal to an autoincrementing integer and value
                 equal to a dict of parameters used to generate this
                 realization.

    weight : dict
             Dictionary with two keys overlap_area, and overlap_percn
             overlap_area returns the area overlaped by both images
             overlap_percn retuns the total percentage of overlap
    """

    def __init__(self, source=None, destination=None):
        self.source = source
        self.destination = destination

        self.homography = None
        self.fundamental_matrix = None
        self.matches = None
        self._subpixel_offsets = None

        self.weight = {}

        self._observers = set()

        # Subscribe the heatlh observer
        self._health = health.EdgeHealth()

    def __repr__(self):
        return """
        Source Image Index: {}
        Destination Image Index: {}
        Available Masks: {}
        """.format(self.source, self.destination, self.masks)

    @property
    def masks(self):
        mask_lookup = {'fundamental': 'fundamental_matrix',
                       'ratio': 'distance_ratio'}
        if not hasattr(self, '_masks'):
            if self.matches is not None:
                self._masks = pd.DataFrame(True, columns=['symmetry'],
                                           index=self.matches.index)
            else:
                self._masks = pd.DataFrame()
        # If the mask is coming form another object that tracks
        # state, dynamically draw the mask from the object.
        for c in self._masks.columns:
            if c in mask_lookup:
                truncated_mask = getattr(self, mask_lookup[c]).mask
                self._masks[c] = False
                self._masks[c].iloc[truncated_mask.index] = truncated_mask
        return self._masks

    @masks.setter
    def masks(self, v):
        column_name = v[0]
        boolean_mask = v[1]
        self.masks[column_name] = boolean_mask

    @property
    def health(self):
        return self._health.health

    def match(self, k=2):
        """
        Given two sets of descriptors, utilize a FLANN (Approximate Nearest
        Neighbor KDTree) matcher to find the k nearest matches.  Nearness is
        the euclidean distance between descriptors.

        The matches are then added as an attribute to the edge object.
        Parameters
        ----------
        k : int
            The number of neighbors to find

        Returns
        -------

        """
        def mono_matches(a, b):
            fl.add(a.descriptors, a.node_id)
            fl.train()
            self._add_matches(fl.query(b.descriptors, b.node_id, k))
            fl.clear()

        fl = FlannMatcher()
        mono_matches(self.source, self.destination)
        mono_matches(self.destination, self.source)

    def _add_matches(self, matches):
        """
        Given a dataframe of matches, either append to an existing
        matches edge attribute or initially populate said attribute.

        Parameters
        ----------
        matches : dataframe
                  A dataframe of matches
        """
        if self.matches is None:
            self.matches = matches
        else:
            df = self.matches
            self.matches = df.append(matches,
                                     ignore_index=True,
                                     verify_integrity=True)

    def symmetry_check(self):
        if hasattr(self, 'matches'):
            mask = od.mirroring_test(self.matches)
            self.masks = ('symmetry', mask)
        else:
            raise AttributeError('No matches have been computed for this edge.')

    def ratio_check(self, clean_keys=[], **kwargs):
        if hasattr(self, 'matches'):

            matches, mask = self._clean(clean_keys)

            self.distance_ratio = od.DistanceRatio(matches)
            self.distance_ratio.compute(mask=mask, **kwargs)

            # Setup to be notified
            self.distance_ratio._notify_subscribers(self.distance_ratio)

            self.masks = ('ratio', self.distance_ratio.mask)
        else:
            raise AttributeError('No matches have been computed for this edge.')

    def compute_fundamental_matrix(self, clean_keys=[], **kwargs):
        """
        Estimate the fundamental matrix (F) using the correspondences tagged to this
        edge.

        Parameters
        ----------
        clean_keys : list
                     Of strings used to apply masks to omit correspondences

        method : {linear, nonlinear}
                 Method to use to compute F.  Linear is significantly faster at
                 the cost of reduced accuracy.

        See Also
        --------
        autocnet.transformation.transformations.FundamentalMatrix
       :
        """
        if not hasattr(self, 'matches'):
            raise AttributeError('Matches have not been computed for this edge')
            return
        matches, mask = self._clean(clean_keys)

        # TODO: Homogeneous is horribly inefficient here, use Numpy array notation
        s_keypoints = self.source.get_keypoint_coordinates(index=matches['source_idx'],
                                                                 homogeneous=True)
        d_keypoints = self.destination.get_keypoint_coordinates(index=matches['destination_idx'],
                                                                homogeneous=True)

        # Replace the index with the matches index.
        s_keypoints.index = matches.index
        d_keypoints.index = matches.index

        self.fundamental_matrix = FundamentalMatrix(np.zeros((3,3)), index=matches.index)
        self.fundamental_matrix.compute(s_keypoints, d_keypoints, **kwargs)

        # Convert the truncated RANSAC mask back into a full length mask
        mask[mask] = self.fundamental_matrix.mask

        # Subscribe the health watcher to the fundamental matrix observable
        self.fundamental_matrix.subscribe(self._health.update)
        self.fundamental_matrix._notify_subscribers(self.fundamental_matrix)

        # Set the initial state of the fundamental mask in the masks
        self.masks = ('fundamental', mask)

    def compute_homography(self, method='ransac', clean_keys=[], pid=None, **kwargs):
        """
        For each edge in the (sub) graph, compute the homography
        Parameters
        ----------
        outlier_algorithm : object
                            An openCV outlier detections algorithm, e.g. cv2.RANSAC

        clean_keys : list
                     of string keys to masking arrays
                     (created by calling outlier detection)
        Returns
        -------
        transformation_matrix : ndarray
                                The 3x3 transformation matrix

        mask : ndarray
               Boolean array of the outliers
        """

        if hasattr(self, 'matches'):
            matches = self.matches
        else:
            raise AttributeError('Matches have not been computed for this edge')

        matches, mask = self._clean(clean_keys)

        s_keypoints = self.source.get_keypoint_coordinates(index=matches['source_idx'])
        d_keypoints = self.destination.get_keypoint_coordinates(index=matches['destination_idx'])

        self.homography = Homography(np.zeros((3,3)), index=self.masks.index)
        self.homography.compute(s_keypoints.values,
                                d_keypoints.values)

        # Convert the truncated RANSAC mask back into a full length mask
        mask[mask] = self.homography.mask
        self.masks = ('ransac', mask)

        # Finalize the array to get custom attrs to propagate
        self.homography.__array_finalize__(self.homography)

    def subpixel_register(self, clean_keys=[], threshold=0.8,
                          template_size=19, search_size=53, max_x_shift=1.0,
                          max_y_shift=1.0, tiled=False, **kwargs):
        """
        For the entire graph, compute the subpixel offsets using pattern-matching and add the result
        as an attribute to each edge of the graph.

        Parameters
        ----------
        clean_keys : list
             of string keys to masking arrays
             (created by calling outlier detection)

        threshold : float
                    On the range [-1, 1].  Values less than or equal to
                    this threshold are masked and can be considered
                    outliers

        upsampling : int
                     The multiplier to the template and search shapes to upsample
                     for subpixel accuracy

        template_size : int
                        The size of the template in pixels, must be odd

        search_size : int
                      The size of the search

        max_x_shift : float
                      The maximum (positive) value that a pixel can shift in the x direction
                      without being considered an outlier

        max_y_shift : float
                      The maximum (positive) value that a pixel can shift in the y direction
                      without being considered an outlier
        """
        matches = self.matches
        for column, default in {'x_offset': 0, 'y_offset': 0, 'correlation': 0, 'reference': -1}.items():
            if column not in self.matches.columns:
                self.matches[column] = default

        # Build up a composite mask from all of the user specified masks
        matches, mask = self._clean(clean_keys)

        # Grab the full images, or handles
        if tiled is True:
            s_img = self.source.geodata
            d_img = self.destination.geodata
        else:
            s_img = self.source.geodata.read_array()
            d_img = self.destination.geodata.read_array()

        source_image = (matches.iloc[0]['source_image'])

        # for each edge, calculate this for each keypoint pair
        for i, (idx, row) in enumerate(matches.iterrows()):
            s_idx = int(row['source_idx'])
            d_idx = int(row['destination_idx'])

            s_keypoint = self.source.get_keypoint_coordinates(s_idx)
            d_keypoint = self.destination.get_keypoint_coordinates(d_idx)

            # Get the template and search window
            s_template = sp.clip_roi(s_img, s_keypoint, template_size)
            d_search = sp.clip_roi(d_img, d_keypoint, search_size)
            try:
                x_offset, y_offset, strength = sp.subpixel_offset(s_template, d_search, **kwargs)
                self.matches.loc[idx, ('x_offset', 'y_offset',
                                       'correlation', 'reference')] = [x_offset, y_offset, strength, source_image]
            except:
                warnings.warn('Template-Search size mismatch, failing for this correspondence point.')
                continue

        # Compute the mask for correlations less than the threshold
        threshold_mask = self.matches['correlation'] >= threshold

        # Compute the mask for the point shifts that are too large
        query_string = 'x_offset <= -{0} or x_offset >= {0} or y_offset <= -{1} or y_offset >= {1}'.format(max_x_shift,
                                                                                max_y_shift)
        sp_shift_outliers = self.matches.query(query_string)
        shift_mask = pd.Series(True, index=self.matches.index)
        shift_mask.loc[sp_shift_outliers.index] = False

        # Generate the composite mask and write the masks to the mask data structure
        mask = threshold_mask & shift_mask
        self.masks = ('shift', shift_mask)
        self.masks = ('threshold', threshold_mask)
        self.masks = ('subpixel', mask)

    def suppress(self, suppression_func=spf.correlation, clean_keys=[], **kwargs):
        """
        Apply a disc based suppression algorithm to get a good spatial
        distribution of high quality points, where the user defines some
        function to be used as the quality metric.

        Parameters
        ----------
        suppression_func : object
                           A function that returns a scalar value to be used
                           as the strength of a given row in the matches data
                           frame.

        suppression_args : tuple
                           Arguments to be passed on to the suppression function

        clean_keys : list
                     of mask keys to be used to reduce the total size
                     of the matches dataframe.
        """
        if not hasattr(self, 'matches'):
            raise AttributeError('This edge does not yet have any matches computed.')

        matches, mask = self._clean(clean_keys)
        domain = self.source.geodata.raster_size

        # Massage the dataframe into the correct structure
        coords = self.source.get_keypoint_coordinates()
        merged = matches.merge(coords, left_on=['source_idx'], right_index=True)
        merged['strength'] = merged.apply(suppression_func, axis=1, args=([self]))

        if not hasattr(self, 'suppression'):
            # Instantiate the suppression object and suppress matches
            self.suppression = od.SpatialSuppression(merged, domain, **kwargs)
            self.suppression.suppress()
        else:
            for k, v in kwargs.items():
                if hasattr(self.suppression, k):
                    setattr(self.suppression, k, v)
            self.suppression.suppress()

        mask[mask] = self.suppression.mask
        self.masks = ('suppression', mask)

    def plot(self, ax=None, clean_keys=[], **kwargs):
        return plot_edge(self, ax=ax, clean_keys=clean_keys, **kwargs)

    def _clean(self, clean_keys, pid=None):
        """
        Given a list of clean keys and a provenance id compute the
        mask of valid matches

        Parameters
        ----------
        clean_keys : list
                     of columns names (clean keys)
        pid : int
              The provenance id of the parameter set to be cleaned.
              Defaults to the last run.

        Returns
        -------
        matches : dataframe
                  A masked view of the matches dataframe

        mask : series
               A boolean series to inflate back to the full match set
        """
        if clean_keys:
            mask = self.masks[clean_keys].all(axis=1)
        else:
            mask = pd.Series(True, self.matches.index)

        return self.matches[mask], mask

    def overlap(self):
        """
        Acts on an edge and returns the overlap area and percentage of overlap
        between the two images on the edge. Data is returned to the
        weight dictionary
        """
        poly1 = self.source.geodata.footprint
        poly2 = self.destination.geodata.footprint

        overlapinfo = cg.two_poly_overlap(poly1, poly2)

        self.weight['overlap_area'] = overlapinfo[1]
        self.weight['overlap_percn'] = overlapinfo[0]

    def coverage(self, clean_keys = []):
        """
        Acts on the edge given either the source node
        or the destination node and returns the percentage
        of overlap covered by the keypoints. Data for the
        overlap is gathered from the source node of the edge
        resulting in a maximum area difference of 2% when compared
        to the destination.

        Returns
        -------
        total_overlap_percentage : float
                                   returns the overlap area
                                   covered by the keypoints
        """
        if self.matches is None:
            raise AttributeError('Edge needs to have features extracted and matched')
            return
        matches, mask = self._clean(clean_keys)
        source_array = self.source.get_keypoint_coordinates(index=matches['source_idx']).values

        source_coords = self.source.geodata.latlon_corners
        destination_coords = self.destination.geodata.latlon_corners

        convex_hull = cg.convex_hull(source_array)

        convex_points = [self.source.geodata.pixel_to_latlon(row[0], row[1]) for row in convex_hull.points[convex_hull.vertices]]
        convex_coords = [(x, y) for x, y in convex_points]

        source_poly = utils.array_to_poly(source_coords)
        destination_poly = utils.array_to_poly(destination_coords)
        convex_poly = utils.array_to_poly(convex_coords)

        intersection_area = cg.get_area(source_poly, destination_poly)

        total_overlap_coverage = (convex_poly.GetArea()/intersection_area)

        return total_overlap_coverage