# Import relevant packages
import time
import numpy as np
import os
import argparse
import h5py # needed for save_weights, fails otherwise
from keras import backend as K
from keras.models import Model, model_from_json
from keras.layers import Dense, Activation, Input, merge
from keras.layers.core import Flatten, Permute, Reshape, Dropout, Lambda, RepeatVector
from keras.layers.wrappers import TimeDistributed
from keras.optimizers import SGD, Adam, Adadelta
from keras.layers.convolutional import Convolution1D, Convolution2D
from keras.regularizers import l2
from keras.utils.np_utils import to_categorical
no_printing = False
try:
    from keras.utils.visualize_util import plot
except:
    no_printing = True
from makeit.utilities.descriptors import edits_to_vectors, oneHotVector # for testing
from makeit.utilities.threadsafe import threadsafe_generator
import rdkit.Chem as Chem

from scipy.sparse import coo_matrix
import makeit.utilities.io.pickle as pickle
import matplotlib
import matplotlib.pyplot as plt    # for visualization
import scipy.stats as ss
import itertools
from makeit.utilities.io.logger import MyLogger
forwardPredictionNetwork_loc = 'forwardPredictionNetwork'
'''

This script builds, trains, tests, and saves a context-dependent reaction
scoring model that works with a pre-generated .h5 dataset. By using a data
generator instead of reading from multiple files, data is pre-loaded for
upcoming minibatches, so the overall training time is substantially reduced.

'''

def msle_of_true(y_true, y_pred):
    '''Custom loss function that uses the mean squared log error in predicted
    yield, assuming that y_pred are unscaled predictions with the true
    outcome in the first index.'''
    return K.square(K.log(K.clip(y_pred[:, 0:1], K.epsilon(), 1.0)) - K.log(K.clip(y_true, K.epsilon(), 1.0)))

def mse_of_true(y_true, y_pred):
    '''Custom loss function that uses the mean squared error in predicted
    yield, assuming that y_pred are unscaled predictions with the true
    outcome in the first index.'''
    return K.square(y_pred[:, 0:1] - y_true)

def build(F_atom = 1, F_bond = 1, N_h1 = 100, N_h2 = 50, N_h3 = 0, inner_act = 'tanh', l2v = 0.01, lr = 0.0003, N_hf = 20, context_weight = 150.0, enhancement_weight = 0.1, optimizer = Adadelta(), extra_outputs = False, TARGET_YIELD = False, absolute_score = False):
    '''
    Builds the feed forward model.

    N_e:  maximum number of edits of each type
    N_h1: number of hidden nodes in first layer
    N_h2: number of hidden nodes in second layer
    inner_act: activation function
    '''

    h_lost    = Input(shape = (None, None, F_atom), name = "H_lost")
    h_gain    = Input(shape = (None, None, F_atom), name = "H_gain")
    bond_lost = Input(shape = (None, None, F_bond), name = "bond_lost")
    bond_gain = Input(shape = (None, None, F_bond), name = "bond_gain")
    reagents  = Input(shape = (256,), name = "reagent_FP") # TODO: remove hard-coded length
    solvent   = Input(shape = (6,), name = "solvent_descriptors_c_e_s_a_b_v")
    temp      = Input(shape = (1,), name = "temperature_C")

    # h_lost_r    = Reshape((h_lost.shape[0] * h_lost.shape[1], F_atom), name = "flatten H_lost")(h_lost)
    # h_gain_r    = Reshape((h_gain.shape[1] * h_gain.shape[1], F_atom), name = "flatten H_gain")(h_gain)
    # bond_lost_r = Reshape((bond_lost.shape[0] * bond_lost.shape[1], F_bond), name = "flatten bond_lost")(bond_lost)
    # bond_gain_r = Reshape((bond_gain.shape[0] * bond_gain.shape[1], F_bond), name = "flatten bond_gain")(bond_gain)

    # Combine along first three dimensions
    dynamic_reshaper       = lambda x: K.reshape(x, (x.shape[0] * x.shape[1] * x.shape[2], x.shape[3]), ndim  = x.ndim-2)
    dynamic_reshaper_shape = lambda x: (None,) + x[3:]

    h_lost_r    = Lambda(dynamic_reshaper, output_shape = dynamic_reshaper_shape, name = "flatten_H_lost")(h_lost)
    h_gain_r    = Lambda(dynamic_reshaper, output_shape = dynamic_reshaper_shape, name = "flatten_H_gain")(h_gain)
    bond_lost_r = Lambda(dynamic_reshaper, output_shape = dynamic_reshaper_shape, name = "flatten_bond_lost")(bond_lost)
    bond_gain_r = Lambda(dynamic_reshaper, output_shape = dynamic_reshaper_shape, name = "flatten_bond_gain")(bond_gain)

    h_lost_h1    = Dense(N_h1, activation = inner_act, W_regularizer = l2(l2v), name = "embed H_lost 1")(h_lost_r)
    h_gain_h1    = Dense(N_h1, activation = inner_act, W_regularizer = l2(l2v), name = "embed H_gain 1")(h_gain_r)
    bond_lost_h1 = Dense(N_h1, activation = inner_act, W_regularizer = l2(l2v), name = "embed bond_lost 1")(bond_lost_r)
    bond_gain_h1 = Dense(N_h1, activation = inner_act, W_regularizer = l2(l2v), name = "embed bond_gain 1")(bond_gain_r)

    N_h = N_h1

    if N_h2 > 0:
        h_lost_h2    = Dense(N_h2, activation = inner_act, W_regularizer = l2(l2v), name = "embed H_lost 2")(h_lost_h1)
        h_gain_h2    = Dense(N_h2, activation = inner_act, W_regularizer = l2(l2v), name = "embed H_gain 2")(h_gain_h1)
        bond_lost_h2 = Dense(N_h2, activation = inner_act, W_regularizer = l2(l2v), name = "embed bond_lost 2")(bond_lost_h1)
        bond_gain_h2 = Dense(N_h2, activation = inner_act, W_regularizer = l2(l2v), name = "embed bond_gain 2")(bond_gain_h1)
        N_h = N_h2

        if N_h3 > 0:
            h_lost_h    = Dense(N_h3, activation = inner_act, W_regularizer = l2(l2v), name = "embed H_lost 3")(h_lost_h2)
            h_gain_h    = Dense(N_h3, activation = inner_act, W_regularizer = l2(l2v), name = "embed H_gain 3")(h_gain_h2)
            bond_lost_h = Dense(N_h3, activation = inner_act, W_regularizer = l2(l2v), name = "embed bond_lost 3")(bond_lost_h2)
            bond_gain_h = Dense(N_h3, activation = inner_act, W_regularizer = l2(l2v), name = "embed bond_gain 3")(bond_gain_h2)
            N_h         = N_h3

        else:
            h_lost_h    = h_lost_h2
            h_gain_h    = h_gain_h2
            bond_lost_h = bond_lost_h2
            bond_gain_h = bond_gain_h2

    else:
        h_lost_h    = h_lost_h1
        h_gain_h    = h_gain_h1
        bond_lost_h = bond_lost_h1
        bond_gain_h = bond_gain_h1

    # Re-expand (using tricky Merge layer, where x[0] is actual data and x[1] is only used for shape)
    dynamic_unreshaper = lambda x: K.reshape(x[0], (x[1].shape[0], x[1].shape[1], x[1].shape[2], x[0].shape[1]), ndim  = x[0].ndim+2)
    dynamic_unreshaper_shape = lambda x: x[1][:3] + x[0][1:]

    h_lost_r2    = Lambda(dynamic_unreshaper, output_shape = dynamic_unreshaper_shape, name = "expand H_lost edits")([h_lost_h, h_lost])
    h_gain_r2    = Lambda(dynamic_unreshaper, output_shape = dynamic_unreshaper_shape, name = "expand H_gain edits")([h_gain_h, h_gain])
    bond_lost_r2 = Lambda(dynamic_unreshaper, output_shape = dynamic_unreshaper_shape, name = "expand bond_lost edits")([bond_lost_h, bond_lost])
    bond_gain_r2 = Lambda(dynamic_unreshaper, output_shape = dynamic_unreshaper_shape, name = "expand bond_gain edits")([bond_gain_h, bond_gain])

    # Add edits within a single candidate
    sum_along_axis2       = lambda x: K.sum(x, axis = 2)
    sum_along_axis2_shape = lambda x: x[:2] + x[3:]
    h_lost_sum    = Lambda(sum_along_axis2, output_shape = sum_along_axis2_shape, name = "sum H_lost")(h_lost_r2)
    h_gain_sum    = Lambda(sum_along_axis2, output_shape = sum_along_axis2_shape, name = "sum H_gain")(h_gain_r2)
    bond_lost_sum = Lambda(sum_along_axis2, output_shape = sum_along_axis2_shape, name = "sum bond_lost")(bond_lost_r2)
    bond_gain_sum = Lambda(sum_along_axis2, output_shape = sum_along_axis2_shape, name = "sum bond_gain")(bond_gain_r2)

    # Sum across edits in their intermediate representation
    try:
        net_sum = merge.concatenate([h_lost_sum, h_gain_sum, bond_lost_sum, bond_gain_sum], name = "concat across edits")
    except AttributeError:
        net_sum = merge([h_lost_sum, h_gain_sum, bond_lost_sum, bond_gain_sum], mode = 'concat', name = "concat across edits")

    feature_to_feature = Dense(N_hf, activation = inner_act, W_regularizer = l2(l2v))
    net_sum_h = TimeDistributed(feature_to_feature, name = "reaction embedding post-sum")(net_sum)

    # Take reagents -> intermediate representation -> cosine similarity to enhance reaction
    reagents_h = Dense(N_hf, activation = inner_act, W_regularizer = l2(l2v), name = "reagent fingerprint to features")(reagents)

    # Trick to repeat reagents using merge layer (so N_c is implicit)
    # x[0] is the original vector and x[1] is just to get the number of candidates (shape)
    context_repeater     = lambda x: K.repeat(x[0], x[1].shape[1])
    context_repeater_shape = lambda x: (x[0][0], x[1][1]) + x[0][1:]
    reagents_h_rpt = Lambda(context_repeater, output_shape = context_repeater_shape, name = "broadcast reagent vector")([reagents_h, h_lost])
    solvent_rpt    = Lambda(context_repeater, output_shape = context_repeater_shape, name = "broadcast solvent vector")([solvent, h_lost])
    temp_rpt       = Lambda(context_repeater, output_shape = context_repeater_shape, name = "broadcast temperature")([temp, h_lost])

    # Dot product between reagents and net_sum_h gives enhancement factor
    try:
        enhancement_mul = merge.multiply([net_sum_h, reagents_h_rpt], name = "multiply reaction with reagents [dot 1/2]")
    except AttributeError:
        enhancement_mul = merge([net_sum_h, reagents_h_rpt], mode = 'mul', name = "multiply reaction with reagents [dot 1/2]")
    enhancement_r = Lambda(lambda x: K.sum(x, axis = -1, keepdims = True), output_shape = lambda x: x[:-1] + (1,), name = "sum reaction with reagents [dot 2/2]")(enhancement_mul)

    # Converge to G0, C[not real], E, S, A, B, V, and K
    feature_to_params = Dense(8, activation = 'linear', W_regularizer = l2(l2v))
    params = TimeDistributed(feature_to_params, name = "features to K,G0,C,E,S,A,B,V")(net_sum_h)

    # Concatenate enhancement and solvents
    try:
        params_enhancement = merge.concatenate([params, enhancement_r, solvent_rpt, temp_rpt], name = "concatenate context")
    except AttributeError:
        params_enhancement = merge([params, enhancement_r, solvent_rpt, temp_rpt], mode = 'concat', name = "concatenate context")

    # # Calculate using thermo-ish
    # # K * exp(- (G0 + delG_solv) / T + enhancement)
    # unscaled_score = Lambda(
    #     lambda x: x[:, :, 0] * K.exp(- (x[:, :, 1] + K.sum(x[:, :, 2:8] * x[:, :, 8:14], axis = -1)) / (x[:, :, 15] + 273.15) + x[:, :, 8]),
    #     output_shape = lambda x: (None, N_c,),
    #     name = "propensity = K * exp(- (G0 + cC + eE + ... + vV) / T + enh.)"
    # )(params_enhancement)

    unscaled_score = Lambda(
            lambda x: x[:, :, 0] - context_weight * (x[:, :, 1] + K.sum(x[:, :, 2:8] * x[:, :, 9:15], axis = -1)) / (x[:, :, 15] + 273.15) + enhancement_weight * x[:, :, 8],
            output_shape = lambda x: x[:2],
            name = "propensity = logK - (G0 + cC + eE + ... + vV) / T + enh."
        )(params_enhancement)

    if absolute_score:
        score = unscaled_score
    elif not TARGET_YIELD:
        score = Activation('softmax', name = "scores to probs")(unscaled_score)
    else:
        scaled_score = Activation(lambda x: K.exp(x - 3.0), name = 'exponential activation')(unscaled_score)
        # Do not scale score with softmax (which would force 100% conversion)
        # Scale linearly
        score = Lambda(
            lambda x: x / K.tile(K.maximum(1.0, K.sum(x, axis = -1, keepdims = True)), (1, x.shape[1])),
        name = "scale if sum(score)>1")(scaled_score)

    #score = unscaled_score_r

    if extra_outputs:
        model = Model(input = [h_lost, h_gain, bond_lost, bond_gain, reagents, solvent, temp],
            output = [h_lost_sum, h_gain_sum, bond_lost_sum, bond_gain_sum, net_sum, net_sum_h, params, unscaled_score, score])
        return model

    model = Model(input = [h_lost, h_gain, bond_lost, bond_gain, reagents, solvent, temp],
        output = [score])

    # model.summary()

    # Now compile
    if not TARGET_YIELD:
        model.compile(loss = 'categorical_crossentropy', optimizer = optimizer,
            metrics = ['accuracy'])
    else:
        model.compile(loss = mse_of_true, optimizer = optimizer)

    return model

@threadsafe_generator
def data_generator(start_at, end_at, batch_size, max_N_c = None, shuffle = False):
    '''This function generates batches of data from the
    pickle file since all the data can't fit in memory.

    The starting and ending indices are specified explicitly so the
    same function can be used for validation data as well

    Input tensors are generated on-the-fly so there is less I/O

    max_N_c is the maximum number of candidates to consider. This should ONLY be used
    for training, not for validation or testing.'''

    def bond_string_to_tuple(string):
        split = string.split('-')
        return (split[0], split[1], float(split[2]))

    fileInfo  = [() for j in range(start_at, end_at, batch_size)] # (filePos, startIndex, endIndex)
    batchDims = [() for j in range(start_at, end_at, batch_size)] # dimensions of each batch
    batchNums = np.array([i for (i, j) in enumerate(range(start_at, end_at, batch_size))]) # list to shuffle later

    # Keep returning forever and ever
    with open(DATA_FPATH, 'rb') as fid:

        # Do a first pass through the data
        legend_data = pickle.load(fid) # first doc is legend

        # Pre-load indeces
        CANDIDATE_EDITS_COMPACT = legend_data['candidate_edits_compact']
        ATOM_DESC_DICT          = legend_data['atom_desc_dict']
        T                       = legend_data['T']
        SOLVENT                 = legend_data['solvent']
        REAGENT                 = legend_data['reagent']
        YIELD                   = legend_data['yield']
        REACTION_TRUE_ONEHOT    = legend_data['reaction_true_onehot']

        for i in range(start_at): pickle.load(fid) # throw away first ___ entries

        for k, startIndex in enumerate(range(start_at, end_at, batch_size)):
            endIndex = min(startIndex + batch_size, end_at)

            # Remember this starting position
            fileInfo[k] = (fid.tell(), startIndex, endIndex)

            N = endIndex - startIndex # number of samples this batch
            # print('Serving up examples {} through {}'.format(startIndex, endIndex))

            docs = [pickle.load(fid) for j in range(startIndex, endIndex)]

            # FNeed to figure out size of padded batch
            N_c = max([len(doc[REACTION_TRUE_ONEHOT]) for doc in docs])
            if type(max_N_c) != type(None): # allow truncation during training
                N_c = min(N_c, max_N_c)
            N_e1 = 1; N_e2 = 1; N_e3 = 1; N_e4 = 1;
            for i, doc in enumerate(docs):
                for (c, edit_string) in enumerate(doc[CANDIDATE_EDITS_COMPACT]):
                    if c >= N_c: break
                    edit_string_split = edit_string.split(';')
                    N_e1 = max(N_e1, edit_string_split[0].count(',') + 1)
                    N_e2 = max(N_e2, edit_string_split[1].count(',') + 1)
                    N_e3 = max(N_e3, edit_string_split[2].count(',') + 1)
                    N_e4 = max(N_e4, edit_string_split[3].count(',') + 1)

            # Remember sizes of x_h_lost, x_h_gain, x_bond_lost, x_bond_gain, reaction_true_onehot
            batchDim = (N, N_c, N_e1, N_e2, N_e3, N_e4)

            # print('The padded sizes of this batch will be: N, N_c, N_e1, N_e2, N_e3, N_e4')
            # print(batchDim)
            batchDims[k] = batchDim

        while True:

            if shuffle: np.random.shuffle(batchNums)

            for batchNum in batchNums:
                (filePos, startIndex, endIndex) = fileInfo[batchNum]
                (N, N_c, N_e1, N_e2, N_e3, N_e4) = batchDims[batchNum]
                fid.seek(filePos)

                N = endIndex - startIndex # number of samples this batch
                # print('Serving up examples {} through {}'.format(startIndex, endIndex))

                docs = [pickle.load(fid) for j in range(startIndex, endIndex)]

                # Initialize numpy arrays for x_h_lost, etc.
                x_h_lost = np.zeros((N, N_c, N_e1, F_atom), dtype=np.float32)
                x_h_gain = np.zeros((N, N_c, N_e2, F_atom), dtype=np.float32)
                x_bond_lost = np.zeros((N, N_c, N_e3, F_bond), dtype=np.float32)
                x_bond_gain = np.zeros((N, N_c, N_e4, F_bond), dtype=np.float32)
                reaction_true_onehot = np.zeros((N, N_c), dtype=np.float32)
                yields = np.zeros((N, 1), dtype=np.float32)

                for i, doc in enumerate(docs):

                    for (c, edit_string) in enumerate(doc[CANDIDATE_EDITS_COMPACT]):
                        if c >= N_c:
                            break

                        edit_string_split = edit_string.split(';')
                        edits = [
                            [atom_string for atom_string in edit_string_split[0].split(',') if atom_string],
                            [atom_string for atom_string in edit_string_split[1].split(',') if atom_string],
                            [bond_string_to_tuple(bond_string) for bond_string in edit_string_split[2].split(',') if bond_string],
                            [bond_string_to_tuple(bond_string) for bond_string in edit_string_split[3].split(',') if bond_string],
                        ]

                        try:
                            edit_h_lost_vec, edit_h_gain_vec, \
                                edit_bond_lost_vec, edit_bond_gain_vec = edits_to_vectors(edits, None, atom_desc_dict = doc[ATOM_DESC_DICT])
                        except KeyError as e: # sometimes molAtomMapNumber not found if hydrogens were explicit
                            continue

                        for (e, edit_h_lost) in enumerate(edit_h_lost_vec):
                            if e >= N_e1: raise ValueError('N_e1 not large enough!')
                            x_h_lost[i, c, e, :] = edit_h_lost
                        for (e, edit_h_gain) in enumerate(edit_h_gain_vec):
                            if e >= N_e2: raise ValueError('N_e2 not large enough!')
                            x_h_gain[i, c, e, :] = edit_h_gain
                        for (e, edit_bond_lost) in enumerate(edit_bond_lost_vec):
                            if e >= N_e3: raise ValueError('N_e3 not large enough!')
                            x_bond_lost[i, c, e, :] = edit_bond_lost
                        for (e, edit_bond_gain) in enumerate(edit_bond_gain_vec):
                            if e >= N_e4: raise ValueRrror('N_e4 not large enough!')
                            x_bond_gain[i, c, e, :] = edit_bond_gain

                    # Add truncated reaction true (eventually will not truncate)
                    if type(max_N_c) == type(None):
                        reaction_true_onehot[i, :len(doc[REACTION_TRUE_ONEHOT])] = doc[REACTION_TRUE_ONEHOT]
                    else:
                        reaction_true_onehot[i, :min(len(doc[REACTION_TRUE_ONEHOT]), max_N_c)] = doc[REACTION_TRUE_ONEHOT][:max_N_c]
                    yields[i, 0] = doc[YIELD] / 100.0

                # Get rid of NaNs
                x_h_lost[np.isnan(x_h_lost)] = 0.0
                x_h_gain[np.isnan(x_h_gain)] = 0.0
                x_bond_lost[np.isnan(x_bond_lost)] = 0.0
                x_bond_gain[np.isnan(x_bond_gain)] = 0.0
                x_h_lost[np.isinf(x_h_lost)] = 0.0
                x_h_gain[np.isinf(x_h_gain)] = 0.0
                x_bond_lost[np.isinf(x_bond_lost)] = 0.0
                x_bond_gain[np.isinf(x_bond_gain)] = 0.0

                # print('Batch {} to {}'.format(startIndex, endIndex))
                # yield (x, y) as tuple, but each one is a list

                if TARGET_YIELD:
                    y = yields
                else:
                    y = reaction_true_onehot

                yield (
                    [
                        x_h_lost,
                        x_h_gain,
                        x_bond_lost,
                        x_bond_gain,
                        np.array([doc[REAGENT] for doc in docs], dtype=np.float32), # reagent
                        np.array([doc[SOLVENT] for doc in docs], dtype=np.float32), # solvent
                        np.array([doc[T] for doc in docs], dtype=np.float32), # temperature
                    ],
                    [
                        y,
                    ],
                )

@threadsafe_generator
def label_generator(start_at, end_at, batch_size):
    '''This function generates labels to match the data generated
    by data_generator'''

    filePos_start_at = -1

    # Keep returning forever and ever
    with open(LABELS_FPATH, 'rb') as fid:
        while True:
            # Is this the first iteration?
            if filePos_start_at == -1:

                # Remember where data starts
                legend_labels = pickle.load(fid) # first doc is legend
                CANDIDATE_SMILES = legend_labels['candidate_smiles']
                CANDIDATE_EDITS  = legend_labels['candidate_edits_compact']
                REACTION_TRUE    = legend_labels['reaction_true']
                RXDID            = legend_labels['rxdid']

                for i in range(start_at): pickle.load(fid) # throw away first ___ entries
                filePos_start_at = fid.tell()

            else:
                fid.seek(filePos_start_at)

            for startIndex in range(start_at, end_at, batch_size):
                endIndex = min(startIndex + batch_size, end_at)

                docs = [pickle.load(fid) for j in range(startIndex, endIndex)]
                yield {
                    'candidate_smiles': [doc[CANDIDATE_SMILES] for doc in docs],
                    'candidate_edits':  [doc[CANDIDATE_EDITS] for doc in docs],
                    'reaction_true':    [doc[REACTION_TRUE] for doc in docs],
                    'rxdid':            [doc[RXDID] for doc in docs]
                }

            filePos_start_at = -1

def get_data(max_N_c = None, shuffle = False):
    '''Creates a dictionary defining data generators for
    training and validation given pickled data/label files

    max_N_c and shuffle only refers to training data'''

    with open(DATA_FPATH, 'rb') as fid:
        legend_data = pickle.load(fid)
    with open(LABELS_FPATH, 'rb') as fid:
        legend_labels = pickle.load(fid)

    N_samples =  legend_data['N_examples']
    N_train = int(N_samples * split_ratio[0])
    N_val = int(N_samples * split_ratio[1])
    N_test = N_samples - N_train - N_val
    print('Total number of samples: {}'.format(N_samples))
    print('Training   on {}% - {}'.format(split_ratio[0]*100, N_train))
    print('Validating on {}% - {}'.format(split_ratio[1]*100, N_val))
    print('Testing    on {}% - {}'.format((1-split_ratio[1]-split_ratio[0])*100, N_test))

    return {
        'N_samples': N_samples,
        'N_train': N_train,
        #
        'train_generator': data_generator(0, N_train, batch_size, max_N_c = max_N_c, shuffle = shuffle),
        'train_label_generator': label_generator(0, N_train, batch_size),
        'train_nb_samples': N_train,
        #
        'val_generator': data_generator(N_train, N_train + N_val, batch_size),
        'val_label_generator': label_generator(N_train, N_train + N_val, batch_size),
        'val_nb_samples': N_val,
        #
        'test_generator': data_generator(N_train + N_val, N_samples, batch_size),
        'test_label_generator': label_generator(N_train + N_val, N_samples, batch_size),
        'test_nb_samples': N_test,
        #
        #
        'batch_size': batch_size,
    }

def train(model, data):
    '''Trains the Keras model'''

    # Add additional callbacks
    from keras.callbacks import ModelCheckpoint, CSVLogger, EarlyStopping
    callbacks = [
        ModelCheckpoint(WEIGHTS_FPATH, save_weights_only = True), # save every epoch
        CSVLogger(HIST_FPATH),
        EarlyStopping(patience = 5),
    ]

    try:
        hist = model.fit_generator(data['train_generator'],
            samples_per_epoch = data['train_nb_samples'],
            nb_epoch = nb_epoch,
            validation_data = data['val_generator'],
            nb_val_samples = data['val_nb_samples'],
            #pickle_safe = True,
            callbacks = callbacks,
            verbose = 1,
        )

    except KeyboardInterrupt:
        print('Stopped training early!')

def test(model, data):
    '''
    Given a trained model and a list of samples, this function tests
    the model
    '''

    print('Testing model')

    fid = open(TEST_FPATH, 'w')
    fid.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format(
        'reaction_smiles', 'train/val',
        'true_edit', 'prob_true_edit',
        'predicted_edit(or no. 2)', 'prob_predicted_edit(or no. 2)',
        'rank_true_edit', 'true_smiles', 'predicted_smiles(or no. 2)',
        'RXD_id','yield_true',
    ))

    def test_on_set(fid, dataset, data_generator, label_generator, num_batches):
        '''Helper function that works for both training and validation sets'''
        print('Testing on {} data'.format(dataset))
        # Need to process data using generator

        our_preds = []
        true_preds = []
        corr = 0

        for batch_num in range(num_batches):
            (x, y) = data_generator.next()
            labels = label_generator.next()
            y = y[0] # only one output, which is True/False or yield

            # TODO: pre-fetch data in queue
            preds = model.predict_on_batch(x)

            for i in range(preds.shape[0]):

                edits = labels['candidate_edits'][i]
                pred = preds[i, :]
                if not TARGET_YIELD:
                    trueprob = pred[y[i,:] != 0][0] # prob assigned to true outcome
                    rank_true_edit = 1 + len(pred) - (ss.rankdata(pred))[np.argmax(y[i,:])]

                    true_preds.append(trueprob)
                    our_preds.append(pred[np.argmax(y[i,:])])
                    if np.argmax(pred) == np.argmax(y[i,:]):
                        corr += 1

                    # Get most informative labels for the highest predictions
                    if rank_true_edit != 1:
                        # record highest probability
                        most_likely_edit_i = np.argmax(pred)
                        most_likely_prob   = np.max(pred)
                    else:
                        # record number two prediction
                        most_likely_edit_i = np.argmax(pred[pred != np.max(pred)])
                        most_likely_prob   = np.max(pred[pred != np.max(pred)])
                    trueyield = float(labels['reaction_true'][i].split(',y:')[1].split('%')[0])/100.0

                else:
                    trueprob = pred[0] # true outcome always at first index
                    trueyield = y[i, 0]
                    rank_true_edit = 1 + len(pred) - (ss.rankdata(pred))[0]
                    if rank_true_edit == 1: corr += 1
                    true_preds.append(trueprob)
                    our_preds.append(pred[0])
                    if rank_true_edit != 1:
                        most_likely_edit_i = np.argmax(pred)
                        most_likely_prob = np.max(pred)
                    else:
                        most_likely_edit_i = np.argmax(pred[1:]) # without first one
                        most_likely_prob = np.max(pred[1:])

                try:
                    most_likely_smiles = labels['candidate_smiles'][i][most_likely_edit_i]
                    most_likely_edit   = edits[most_likely_edit_i]
                except IndexError:
                    most_likely_smiles = 'no_reaction'
                    most_likely_edit   = 'no_reaction'

                fid.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format(
                    labels['reaction_true'][i], dataset,
                    edits[np.argmax(y[i,:])], trueprob,
                    most_likely_edit, most_likely_prob,
                    rank_true_edit, labels['reaction_true'][i].split('>')[-1],
                    most_likely_smiles, labels['rxdid'][i], trueyield
                ))

        return our_preds, corr

    train_preds, train_corr = test_on_set(fid, 'train', data['train_generator'],
        data['train_label_generator'],
        int(np.ceil(data['train_nb_samples']/float(data['batch_size'])))
    )
    val_preds, val_corr = test_on_set(fid, 'val', data['val_generator'],
        data['val_label_generator'],
        int(np.ceil(data['val_nb_samples']/float(data['batch_size'])))
    )
    test_preds, test_corr = test_on_set(fid, 'test', data['test_generator'],
        data['test_label_generator'],
        int(np.ceil(data['test_nb_samples']/float(data['batch_size'])))
    )

    fid.close()

    train_acc = train_corr / float(len(train_preds))
    val_acc = val_corr / float(len(val_preds))
    test_acc = test_corr / float(len(test_preds))

    train_preds = np.array(train_preds)
    val_preds = np.array(val_preds)
    test_preds = np.array(test_preds)

    def histogram(array, title, path, acc):
        acc = int(acc * 1000)/1000. # round3
        try:
            # Visualize in histogram
            weights = np.ones_like(array) / len(array)
            plt.clf()
            n, bins, patches = plt.hist(array, np.arange(0, 1.02, 0.02), facecolor = 'blue', alpha = 0.5, weights = weights)
            plt.xlabel('Assigned probability to true product')
            plt.ylabel('Normalized frequency')
            plt.title('Histogram of pseudo-probabilities - {} (N={},acc={})'.format(title, len(array), acc))
            plt.axis([0, 1, 0, 1])
            plt.grid(True)
            plt.savefig(path, bbox_inches = 'tight')
        except:
            pass

    histogram(train_preds, 'TRAIN', HISTOGRAM_FPATH % 'train', train_acc)
    histogram(val_preds, 'VAL', HISTOGRAM_FPATH % 'val', val_acc)
    histogram(test_preds, 'TEST', HISTOGRAM_FPATH % 'test', test_acc)

if __name__ == '__main__':

    np.random.seed(0)

    parser = argparse.ArgumentParser()
    parser.add_argument('--nb_epoch', type = int, default = 100,
                        help = 'Number of epochs to train for, default 100')
    parser.add_argument('--batch_size', type = int, default = 20,
                        help = 'Batch size, default 20')
    parser.add_argument('--Nh1', type = int, default = 40,
                        help = 'Number of hidden nodes in first layer, default 40')
    parser.add_argument('--Nh2', type = int, default = 0,
                        help = 'Number of hidden nodes in second layer, default 0')
    parser.add_argument('--Nh3', type = int, default = 0,
                        help = 'Number of hidden nodes in third layer, ' +
                                'immediately before summing, default 0')
    parser.add_argument('--Nhf', type = int, default = 20,
                        help = 'Number of hidden nodes in layer between summing ' +
                                'and final score, default 20')
    parser.add_argument('--tag', type = str, default = str(time.time()),
                        help = 'Tag for this model')
    parser.add_argument('--retrain', type = bool, default = False,
                        help = 'Retrain with loaded weights, default False')
    parser.add_argument('--test', type = bool, default = False,
                        help = 'Test model only, default False')
    parser.add_argument('--l2', type = float, default = 0.01,
                        help = 'l2 regularization parameter for each Dense layer, default 0.01')
    parser.add_argument('--data_tag', type = str, default = 'makeit/predict/data_edits_reaxys/reaxys',
                        help = 'Data file tag, default makeit/predict/data_edits_reaxys/reaxys')
    parser.add_argument('--lr', type = float, default = 0.01,
                        help = 'Learning rate, default 0.01')
    # parser.add_argument('--dr', type = float, default = 0.5,
    #                     help = 'Dropout rate, default 0.5')
    # parser.add_argument('--fold', type = int, default = 5,
    #                     help = 'Which fold of the 5-fold CV is this? Defaults 5')
    parser.add_argument('--visualize', type = bool, default = False,
                help = 'Whether or not to visualize weights ONLY, default False')
    parser.add_argument('--context_weight', type = float, default = 100.0,
                    help = 'Weight assigned to contextual effects, default 100.0')
    parser.add_argument('--enhancement_weight', type = float, default = 0.1,
            help = 'Weight assigned to enhancement factor, default 0.1')
    parser.add_argument('--Nc', type = int, default = 1000,
            help = 'Number of candidates to truncate to during training, default 1000')
    parser.add_argument('--optimizer', type = str, default = 'adadelta',
            help = 'Optimizer to use, default adadelta')
    parser.add_argument('--inner_act', type = str, default = 'tanh',
            help = 'Inner activation function, default "tanh" ')
    parser.add_argument('--yd', type = int, default = 0,
            help = 'Are we targeting yield? 0 or 1, default 0')

    args = parser.parse_args()

    nb_epoch           = int(args.nb_epoch)
    batch_size         = int(args.batch_size)
    N_h1               = int(args.Nh1)
    N_h2               = int(args.Nh2)
    N_h3               = int(args.Nh3)
    N_hf               = int(args.Nhf)
    l2v                = float(args.l2)
    lr                 = float(args.lr)
    max_N_c            = int(args.Nc) # number of candidate edit sets
    context_weight     = float(args.context_weight)
    enhancement_weight = float(args.enhancement_weight)
    optimizer          = args.optimizer
    inner_act          = args.inner_act
    TARGET_YIELD       = bool(args.yd)

    # THIS_FOLD_OUT_OF_FIVE = int(args.fold)
    tag = args.tag

    split_ratio = (0.8, 0.1) # 80% training, 10% validation, balance testing

    if optimizer == 'sgd':
        opt  = SGD(lr = lr, decay = 1e-4, momentum = 0.9)
    elif optimizer == 'adam':
        opt = Adam(lr = lr)
    elif optimizer == 'adadelta':
        opt = Adadelta()
        print('Because Adadelta was selected, ignoring lr setting')
    else:
        raise ValueError('Unrecognized optimizer')

    # Labels
    FROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'output')
    FROOT = os.path.join(FROOT, tag)
    if not os.path.isdir(FROOT):
        print(FROOT)
        os.mkdir(FROOT)
    MODEL_FPATH = os.path.join(FROOT, 'model.json')
    WEIGHTS_FPATH = os.path.join(FROOT, 'weights.h5')
    HIST_FPATH = os.path.join(FROOT, 'hist.csv')
    TEST_FPATH = os.path.join(FROOT, 'probs.dat')
    HISTOGRAM_FPATH = os.path.join(FROOT, 'histogram %s.png')
    ARGS_FPATH = os.path.join(FROOT, 'args.json')

    with open(ARGS_FPATH, 'w') as fid:
        import json
        json.dump(args.__dict__, fid)

    DATA_FPATH = '{}_data.pickle'.format(args.data_tag)
    LABELS_FPATH = '{}_labels.pickle'.format(args.data_tag)

    this_dir = os.getcwd()
    mol = Chem.MolFromSmiles('[CH3:1][CH3:2]')
    (a, _, b, _) = edits_to_vectors((['1'],[],[('1','2',1.0)],[]), mol)
    os.chdir(this_dir)

    F_atom = len(a[0])
    F_bond = len(b[0])

    if bool(args.retrain):
        print('Reloading from file')
        rebuild = raw_input('Do you want to rebuild from scratch instead of loading from file? [n/y] ')
        if rebuild == 'y':
            model = build(F_atom = F_atom, F_bond = F_bond, N_h1 = N_h1,
                N_h2 = N_h2, N_h3 = N_h3, N_hf = N_hf, l2v = l2v, lr = lr, optimizer = opt, inner_act = inner_act,
                context_weight = context_weight, enhancement_weight = enhancement_weight, extra_outputs = bool(args.visualize),
                TARGET_YIELD = TARGET_YIELD)
        else:
            model = model_from_json(open(MODEL_FPATH).read())
            if TARGET_YIELD:
                model.compile(loss = mse_of_true)
            else:
                model.compile(loss = 'categorical_crossentropy',
                optimizer = opt,
                metrics = ['accuracy'])
        model.load_weights(WEIGHTS_FPATH)
    else:
        model = build(F_atom = F_atom, F_bond = F_bond, N_h1 = N_h1, N_h2 = N_h2, N_h3 = N_h3, N_hf = N_hf, l2v = l2v, lr = lr, context_weight = context_weight, inner_act = inner_act,
            enhancement_weight = enhancement_weight, optimizer = opt)
        try:
            with open(MODEL_FPATH, 'w') as outfile:
                outfile.write(model.to_json())
        except:
            print('could not write model to json')

    if bool(args.test):
        data = get_data(max_N_c = max_N_c, shuffle = False)
        test(model, data)
        quit(1)

    if bool(args.visualize):
        batch_size = 1
        data = get_data(max_N_c = max_N_c, shuffle = False)
        data_generator = data['test_generator']
        label_generator = data['test_label_generator']
        ex = 0
        while True:
            (x, y) = data_generator.next()
            labels = label_generator.next()

            z = model.predict(x)

            for i, zz in enumerate(z):
                plt.clf()
                if len(zz.shape) == 3 and zz.shape[0] == 1:
                    zz = zz[0]
                plt.pcolor(zz)
                plt.colorbar()
                plt.tight_layout()
                plt.savefig(os.path.join(FROOT, 'ex{}_output{}'.format(ex, i)))
            with open(os.path.join(FROOT, 'ex{}.info'.format(ex)), 'w') as fid:
                fid.write('{}'.format(labels))
            ex += 1
            raw_input('Pause...')

        quit(1)

    data = get_data(max_N_c = max_N_c, shuffle = True)
    train(model, data)
    model.save_weights(WEIGHTS_FPATH, overwrite = True)
    data = get_data(max_N_c = max_N_c, shuffle = False)

    test(model, data)


import numpy as np
import cv2
import time
import matplotlib.pyplot as plt
from terri_utils import img_transform
from terri_utils import round2nearest_multiple
# For DC motor
from adafruit_servokit import ServoKit # seteja el pinout a 1000 (GPIO.TEGRA_SOC)
import RPi.GPIO as GPIO

from processing_unit import GeomPU
# For Servo motor
import board
import busio

import jetson.inference
import jetson.utils
import math
import argparse
import sys

from segnet_utils import *

"""
sudo python3 prova_def_3.py --input-flip=rotate-180 --input-width=640 --input-height=512 --visualize mask --stats true

"""

# HW
output_pin = "GPIO_PE6"
if output_pin is None:
    raise Exception('PWM not supported on this board')

in1_pin = "UART2_RTS"
in2_pin = "DAP4_SCLK"

# Pin Setup:
# Board pin-numbering scheme
#GPIO.setmode(GPIO.BOARD)
def set_movement(in1, in2, direction='f'):
    if direction == 'f':
        GPIO.output(in1, GPIO.LOW)
        GPIO.output(in2, GPIO.HIGH)
    else:
        GPIO.output(in1, GPIO.HIGH)
        GPIO.output(in2, GPIO.LOW)

# set pin as an output pin with optional initial state of HIGH
GPIO.setup(output_pin, GPIO.OUT, initial=GPIO.LOW)
p = GPIO.PWM(output_pin,50)
val = 0
p.start(val)
# Configure gpio pins 11 12
GPIO.setup(in1_pin, GPIO.OUT, initial=GPIO.LOW)
GPIO.setup(in2_pin, GPIO.OUT, initial=GPIO.HIGH)
#print("PWM running. Press CTRL+C to exit.")

# On the Jetson Nano
# Bus 0 (pins 28,27) is board SCL_1, SDA_1 in the jetson board definition file
# Bus 1 (pins 5, 3) is board SCL, SDA in the jetson definition file
# Default is to Bus 1; We are using Bus 0, so we need to construct the busio first ...
#print("Initializing Servos")
i2c_bus0=(busio.I2C(board.SCL_1, board.SDA_1))
#print("Initializing ServoKit")
kit = ServoKit(channels=16, i2c=i2c_bus0)
# kit[0] is the bottom servo
# kit[1] is the top servo
set_movement(in1_pin, in2_pin, 'f')
kit.servo[0].angle = int(18)
p.ChangeDutyCycle(0)
print("Done initializing")

# Disconnect
print("Aneu desconectant")
for j in range(45):
    print(45-j)
    #time.sleep(1)

def eulerAnglesToRotationMatrix(theta):
    """
    https://math.stackexchange.com/questions/2796055/3d-coordinate-rotation-using-roll-pitch-yaw
    """
    R_x = np.array([[1,         0,                  0                   ],
                    [0,         math.cos(theta[0]), -math.sin(theta[0]) ],
                    [0,         math.sin(theta[0]), math.cos(theta[0])  ]
                    ])

    R_y = np.array([[math.cos(theta[1]),    0,      math.sin(theta[1])  ],
                    [0,                     1,      0                   ],
                    [-math.sin(theta[1]),   0,      math.cos(theta[1])  ]
                    ])

    R_z = np.array([[math.cos(theta[2]),    -math.sin(theta[2]),    0],
                    [math.sin(theta[2]),    math.cos(theta[2]),     0],
                    [0,                     0,                      1]
                    ])

    R = np.dot(R_z, np.dot( R_y, R_x ))

    return R

grid_params = {'xw':0.6, 'yw':+0.25, 'length_x':1.0, 'length_y':0.50, 'num_x':7,'num_y':7}
K = np.array([[676.74,0,317.4],[0,863.29,252.459],[0,0,1]])
angles = np.array([90,90,0])*np.pi/180
R = eulerAnglesToRotationMatrix(angles) # Rotation matrix  WCS->CCS
T = np.array([[0.0],[-0.170],[0.0]]) # WCS expressed in camera coordinate system , before 0.15
Rt = np.hstack((R,T))
geompu = GeomPU(K, Rt,grid_params)

parser = argparse.ArgumentParser(description="terrinus first scripts", formatter_class = argparse.RawTextHelpFormatter, epilog=jetson.utils.videoSource.Usage())
parser.add_argument("--network", type=str, default="fcn-resnet18-voc", help="pre-trained model to load, see below for options")
parser.add_argument("--filter-mode", type=str, default="linear", choices=["point", "linear"], help="filtering mode used during visualization, options are:\n  'point' or 'linear' (default: 'linear')")
parser.add_argument("--visualize", type=str, default="overlay,mask", help="Visualization options (can be 'overlay' 'mask' 'overlay,mask'")
parser.add_argument("--ignore-class", type=str, default="void", help="optional name of class to ignore in the visualization results (default: 'void')")
parser.add_argument("--alpha", type=float, default=150.0, help="alpha blending value to use during overlay, between 0.0 and 255.0 (default: 150.0)")
parser.add_argument("--stats", action="store_true", help="compute statistics about segmentation mask class output")
opt = parser.parse_known_args()[0]

camera = jetson.utils.videoSource('csi://0', argv=sys.argv)
net = jetson.inference.segNet('fcn-resnet18-sun-640x512')
buffers = segmentationBuffers(net,opt)

# Disconnect
print("Aneu desconectant")
for j in range(45):
    print(45-j)
    time.sleep(1)

try:

    iterations = 0
    while(iterations<700):
        image = camera.Capture()
        buffers.Alloc(image.shape, image.format)
        net.Process(image)
        net.Mask(buffers.mask, width=640, height=512)
        mask = buffers.mask
        mask_np = jetson.utils.cudaToNumpy(mask)
        pred = mask_np[:,:,1]==128 # from segnet-console2.md jetson inference
        pred = np.uint8(pred)

        #cv2.imwrite("/home/dlinano/Pictures/exp_1_pred/{:03d}.jpeg".format(iterations), pred)
        BEV_img = geompu.create_BEV_image(pred)

        throttle_map, trajectory_BEV, trajectory_img = geompu.evaluate_grid(BEV_img)
        throttle_weights = np.array([0.1,0.1,0.1,0.1,0.2,0.2,0.2])
        max_throt_val = 75
        final_throttle = (max_throt_val/grid_params['num_y'])*np.sum(throttle_map*throttle_weights)
        steering_weights = np.array([0.1,0.1,0.1,0.1,0.2,0.2,0.2])
        h,w = BEV_img.shape
        steering_ys = (trajectory_BEV[1,:]/w)*36.0
        final_steering = np.sum(steering_weights*steering_ys)

        if final_steering > 18:
            final_steering = final_steering*1.2
        elif final_steering < 18:
            final_steering = final_steering*0.8
        kit.servo[0].angle = int(final_steering)
        p.ChangeDutyCycle(final_throttle)
        iterations = iterations + 1

finally:
    camera.Close()
    p.stop()
    GPIO.output(in2_pin, GPIO.LOW)
    GPIO.cleanup()


def get_db_columns(db_class):
    """
    This function returns a dictionary where the keys are the columns of
    the table corresponding to the db_class and the values are the column
    properties such as type, is_foreign_key and if so, the related table
    and column.

    The function applies only to the sqlalchemy backend and for other
    backends it requires a mapping sqlalchemy-->new_backend. For Django this
    mapping is the dummy_model.

    A similar logic applies to the QueryBuilder that is entirely built upon
    SQLAlchemy (+ dummy model, for Django)

    :param db_class: the database model whose schema has to be returned
    :return: a dictionary
    """

    ## Retrieve the columns of the table corresponding to the present class
    # and its foreignkeys
    table = db_class.metadata.tables[db_class.__tablename__]

    # Here we check both columns, column properties, and hybrid properties
    from sqlalchemy.orm import class_mapper
    from sqlalchemy.orm.properties import ColumnProperty

    from sqlalchemy.ext.hybrid import hybrid_property

    # column_properties = [_ for _ in class_mapper(db_class).iterate_properties
    column_properties = [_ for _ in class_mapper(db_class).all_orm_descriptors
                         if isinstance(_, ColumnProperty)]

    hybrid_properties = [_ for _ in class_mapper(db_class).all_orm_descriptors
                         if isinstance(_, hybrid_property)]

    # Ordinary columns
    columns = table.columns

    # Determine the keys (for hybrid_properties I rely on __name__)
    column_property_keys = map(lambda x: x.key, column_properties)
    hybrid_property_keys = map(lambda x: x.__name__, hybrid_properties)
    column_keys = map(lambda x: x.key, columns)

    # Check whether properties contain objects that are not columns
    property_keys = [_ for _ in column_property_keys if _ not in column_keys]

    column_types = map(lambda x: x.type, columns.values())

    # Assume None for the time being for column_property and hybrid_property
    # types
    # TODO find a way to assess the type
    column_property_types = [None] * len(property_keys)
    hybrid_property_types = [None] * len(hybrid_property_keys)

    foreign_keys = [get_foreign_key_infos(foreign_key) for foreign_key in
                    table.foreign_keys]

    ## merge first column_keys and than column_property_keys
    column_names = column_keys + property_keys + hybrid_property_keys
    column_types.extend(column_property_types)
    column_types.extend(hybrid_property_types)

    column_python_types = []

    from sqlalchemy.dialects.postgresql import UUID, JSONB

    for column_type in column_types:
        # Treat the case where there is no natural python_type
        # counterpart to the column type (specifically because of usage
        # of sqlalchemy dialect)
        if column_type is not None:
            try:
                column_python_types.append(column_type.python_type)
            except NotImplementedError:
                if isinstance(column_type, UUID):
                    column_python_types.append(unicode)
                elif isinstance(column_type, JSONB):
                    column_python_types.append(dict)
                else:
                    raise NotImplementedError("Unknown type from the "
                                              "database schema: {}".format(
                        column_type))
        else:
            column_python_types.append(None)

    ## Fill in the returned dictionary
    schema = {}

    # Fill in the keys based on the column names and the types. By default we
    #  assume that columns are no foreign keys
    for k, v in iter(zip(column_names, column_python_types)):
        schema[k] = {'type': v, 'is_foreign_key': False}

    # Add infos about the foreign relationships
    for k, referred_table_name, referred_field_name in foreign_keys:
        schema[k].update({
            'is_foreign_key': True,
            'related_table': referred_table_name,
            'related_column': referred_field_name,
        })

    return schema

def get_foreign_key_infos(foreign_key):
    """
    takes a foreignkey sqlalchemy object and returns the referent column
    name and the referred relation and column names
    :param foreign_key: a sqlalchemy ForeignKey object
    :return: a tuple of strings
    """
    referent_column_name = foreign_key.parent.name
    (referred_table_name, referred_field_name) = tuple(
        foreign_key.target_fullname.split('.'))
    return (referent_column_name, referred_table_name, referred_field_name)


import Pyro4
from job_factory import JobFactory

class Server:

    def __init__(self):
        daemon = Pyro4.Daemon()
        ns = Pyro4.locateNS()

        factory = JobFactory()

        uri = daemon.register(factory)
        ns.register("factory", uri)

        print("Server is ready.")
        daemon.requestLoop()

def main():
    Server()

if __name__ == '__main__':
    main()


from collections import deque
idx = [
    [1, 3], [0, 2, 4], [1, 5], [0, 4, 6], [1, 3, 5, 7], [2, 4, 8], [3, 7], [4, 6, 8], [5, 7]
]
P = '123456780'

def bfs():
    q = deque()
    zero_idx = -1
    for i in range(9):
        if board[i] == '0':
            zero_idx = i
            break
    x = ''.join(board)
    check = {x: 1}
    q.append([x, zero_idx, 0])
    while q:
        s_num, zi, cost = q.popleft()
        step = check.get(s_num)
        if s_num == P:
            return cost
        for i in idx[zi]:
            t = list(s_num)
            t[i], t[zi] = t[zi], t[i]
            t = ''.join(t)
            if not check.get(t):
                q.append([t, i, cost + 1])
                check[t] = step
    return -1

board = []
for _ in range(3):
    board.extend(list(input().split()))
print(bfs())


from typing import List

class Solution:
    def threeSumClosest(self, nums: List[int], target: int) -> int:
        result_sum = nums[0] + nums[1] + nums[2]
        nums.sort()
        for x in range(len(nums)):
            i = x + 1
            j = len(nums) - 1
            while i < j:
                cur_sum = nums[x] + nums[i] + nums[j]
                cur_diff = abs(cur_sum - target)
                diff = abs(result_sum - target)
                if cur_diff < diff:
                    result_sum = cur_sum
                if cur_sum - target > 0:
                    j -= 1
                elif cur_sum - target < 0:
                    i += 1
                else:
                    return result_sum
        return result_sum

if __name__ == '__main__':
    solution = Solution()
    print(solution.threeSumClosest([-3,-2,-5,3,-4], -1))


from ZenPacks.zenoss.ZenPackLib.tests import ZPLBaseTestCase

YAML_DOC = """
name: ZenPacks.zenoss.GraphPointLegends
device_classes:
  /Devices:
    templates:
      TEST:
        datasources:
          A:
            type: SNMP
            severity: 5
            datapoints:
              A: {}
            oid: .1.3.6.1.4.1.232.6.2.6.8.1.4
        thresholds:
          a:
            type: MinMaxThreshold
            dsnames: [A_A]
          b:
            type: MinMaxThreshold
            dsnames: [A_A]
          c:
            type: MinMaxThreshold
            dsnames: [A_A]
        graphs:
          Test:
            graphpoints:
              A:
                dpName: A_A
                includeThresholds: true
                thresholdLegends:
                  a:
                    legend: test_a
                    color: 234234
                  b:
                    legend: test_b
                  c:
                    color: AAAAAA
"""

EXPECTED = {'A': {'color': '', 'legend': '${graphPoint/id}'},
            'a': {'color': '234234', 'legend': 'test_a'},
            'b': {'color': '', 'legend': 'test_b'},
            'c': {'color': 'AAAAAA', 'legend': '${graphPoint/id}'}}

class TestThresholdGraphPointLegends(ZPLBaseTestCase):
    """Test Threshold GraphPoint legend and coloring"""

    yaml_doc = YAML_DOC

    def test_threshold_graphpoint(self):
        ''''''
        config = self.configs.get('ZenPacks.zenoss.GraphPointLegends')
        cfg = config.get('cfg')
        tspec = cfg.device_classes.get('/Devices').templates.get('TEST')
        # template based on original spec
        template = tspec.create(self.dmd, False)
        g = template.graphDefs()[0]
        actual = {}
        for gp in g.graphPoints():
            actual[gp.id] = {'legend': gp.legend, 'color': gp.color}
        self.assertEqual(actual, EXPECTED,
                         'Threshold GraphPoint legend/color testing failed, '\
                         'expected {} got {}'.format(EXPECTED, actual))

def test_suite():
    """Return test suite for this module."""
    from unittest import TestSuite, makeSuite
    suite = TestSuite()
    suite.addTest(makeSuite(TestThresholdGraphPointLegends))
    return suite

if __name__ == "__main__":
    from zope.testrunner.runner import Runner
    runner = Runner(found_suites=[test_suite()])
    runner.run()


from django.shortcuts import render, redirect
from .forms import ProductForm
from .models import Product
from .filters import ListingFilter
# Create your views here.
def product_list(request):
    listings= Product.objects.all()
    listing_filter=ListingFilter(request.GET,queryset=listings)
    context={
        'listing_filter':listing_filter
    }
    return render(request,'product/product_list.html',context)

def product_form(request,id=0):
    if request.method=="GET":
        if id==0:
            form= ProductForm()
        else:
            product= Product.objects.get(pk=id)
            form =  ProductForm(instance=product)
        return render(request,'product/product_form.html', {'form':form})
    else:
        if id==0:
            form = ProductForm(request.POST,request.FILES)
        else:
            product= Product.objects.get(pk=id)
            form= ProductForm(request.POST,request.FILES,instance=product)
        if form.is_valid():
            form.save()
        return redirect('/product/list')

def product_delete(request,id):
    product= Product.objects.get(pk=id)
    product.delete()
    return redirect('/product/list')


#!/usr/bin/python3
""" 101-search_twitter.py -  """

import base64
from requests import get, post
from sys import argv

"""
$ curl --request GET
 --url 'https://api.twitter.com/1.1/search/tweets.json?q=nasa&result_type=
popular' --header 'authorization: OAuth oauth_consumer_key=
"consumer-key-for-app",
 oauth_nonce="generated-nonce", oauth_signature="generated-signature",
 oauth_signature_method="HMAC-SHA1", oauth_timestamp="generated-timestamp",
 oauth_token="access-token-for-authed-user", oauth_version="1.0"'
"""

if __name__ == "__main__":

    if len(argv) == 4:
        urlPost = "https://api.twitter.com/oauth2/token HTTP/1.1/"
        baseURL = "https://api.twitter.com/1.1/search/tweets.json/"
        apiKey = argv[1]
        apiSecret = argv[2]
        apiSearch = argv[3]

        authText = apiKey + ':' + apiSecret
        encoded = base64.b64encode(authText.encode('ascii'))

        header = {}
        header['Authorization'] = 'Basic '.encode('ascii') + encoded
        header['Content-Type'] = 'application/x-www-form-urlencoded;'
        header['Content-Type'] += 'charset=UTF-8'
        bodytext = 'grant_type=client_credentials'

        param = {}
        param['q'] = '@feliciahsiehsw'
        param['count'] = 5
        r = post(urlPost, data=param, headers=header)
        print(r.text)
        # r = get(baseURL_auth, params=param))
        # rj = r.json()
        # try:
        #    print("rj: {}".format(rjson.text))
        #    id = rj['statuses']['id_str']
        #     tweetText = rj['statuses']['text']
        #     name = rj['statuses']['user']['name']
        #     print("[{}]] {} by {}".format(id, tweetText, name))
        # except Exception as e:
        #    print('None')
    else:
        print("Usage: ./101-search_twitter.py [API Key] [API Secret] [Search]")


#!/usr/bin/env python3

from __future__ import print_function
from six.moves import input

import sys
import copy
import rospy
import moveit_commander
import moveit_msgs.msg
import geometry_msgs.msg

try:
    from math import pi, tau, dist, fabs, cos
except:  # For Python 2 compatibility
    from math import pi, fabs, cos, sqrt

    tau = 2.0 * pi

    def dist(p, q):
        return sqrt(sum((p_i - q_i) ** 2.0 for p_i, q_i in zip(p, q)))

from std_msgs.msg import String
from moveit_commander.conversions import pose_to_list

# First initialize moveit_commander and a rospy node
moveit_commander.roscpp_initialize(sys.argv)
rospy.init_node("move_group_python_interface_tutorial", anonymous=True) #Kann den Node Namen später noch abändern

# Instantiate a RobotCommander object
robot = moveit_commander.RobotCommander()

# Instantiate a PlanningSceneInterface object
scene = moveit_commander.PlanningSceneInterface()

# Instantiate a MoveGroupCommander object
group_name = "manipulator"
move_group = moveit_commander.MoveGroupCommander(group_name)

# Create a DisplayTrajectory ROS publisher which is used to display trajectories in Rviz
display_trajectory_publisher = rospy.Publisher(
    "/move_group/display_planned_path",
    moveit_msgs.msg.DisplayTrajectory,
    queue_size=20,
)

# Getting Basic Information

# We can get the name of the reference frame for this robot:
planning_frame = move_group.get_planning_frame()
print("============ Planning frame: %s" % planning_frame)

# We can also print the name of the end-effector link for this group:
eef_link = move_group.get_end_effector_link()
print("============ End effector link: %s" % eef_link)

# We can get a list of all the groups in the robot:
group_names = robot.get_group_names()
print("============ Available Planning Groups:", robot.get_group_names())

# Sometimes for debugging it is useful to print the entire state of the
# robot:
print("============ Printing robot state")
print(robot.get_current_state())
print("")

# We get the joint values from the group and change them to the 'test' position values:
joint_goal = move_group.get_current_joint_values()
joint_goal[0] = -1.7453  # shoulder_pan_joint
joint_goal[1] = -2.4435  # shoulder_lift_joint #-2.1991
joint_goal[2] = -1.0996  # elbow_joint
joint_goal[3] = -0.1047  # wrist_1_joint
joint_goal[4] = 1.6057   # wrist_2_joint
joint_goal[5] = 0        # wrist_3_joint

# Drucken Sie den joint_goal, um sicherzustellen, dass die Werte korrekt sind
print("Geplanter joint_goal:", joint_goal)

# The go command can be called with joint values, poses, or without any
# parameters if you have already set the pose or joint target for the group
#move_group.go(joint_goal, wait=True)

# Calling ``stop()`` ensures that there is no residual movement
#move_group.stop()

#----------Der obere Teil war dazu da eine Position test über die Python-API anzufahren. Der untere Teil um die geplante Bewegung in Rviz zu zeigen-------------

#Einstellen der Geschwindigkeit und Beschleunigung
move_group.set_max_velocity_scaling_factor(0.5)
move_group.set_max_acceleration_scaling_factor(0.5)

# Zielwerte setzen
move_group.set_joint_value_target(joint_goal)

planning_result = move_group.plan()
if planning_result[0]:
    plan = planning_result[1]
    print("Plan erfolgreich erstellt!")
else:
    print("Planung fehlgeschlagen!")
    sys.exit()

# Anzeigen der Trajektorie in RViz
display_trajectory = moveit_msgs.msg.DisplayTrajectory()
display_trajectory.trajectory_start = robot.get_current_state()
display_trajectory.trajectory.append(plan)
# Veröffentliche die Trajektorie
display_trajectory_publisher.publish(display_trajectory)

# Gib RViz etwas Zeit, um die Trajektorie zu visualisieren. Passe die Schlafzeit bei Bedarf an.
rospy.sleep(5)

# Frage den Benutzer, ob er den Plan ausführen möchte
input("Drücke Enter, um den Plan auszuführen oder Ctrl+C zum Abbrechen...")

# Ausführen des Plans
move_group.execute(plan, wait=True)


import caffe

def parseProtoString(s):
	from google.protobuf import text_format
	from caffe.proto import caffe_pb2 as pb
	proto_net = pb.NetParameter()
	text_format.Merge(s, proto_net)
	return proto_net

def get_param(l, exclude=set(['top', 'bottom', 'name', 'type'])):
	if not hasattr(l,'ListFields'):
		if hasattr(l,'__delitem__'):
			return list(l)
		return l
	r = dict()
	for f, v in l.ListFields():
		if f.name not in exclude:
			r[f.name] = get_param(v, [])
	return r

class ProtoDesc:
	def __init__(self, prototxt):
		from os import path
		self.prototxt = prototxt
		self.parsed_proto = parseProtoString(open(self.prototxt, 'r').read())
		# Guess the input dimension
		self.input_dim = (3, 227, 227)
		net = self.parsed_proto
		if len(net.input_dim) > 0:
			self.input_dim = net.input_dim[1:]
		else:
			lrs = net.layer
			cs = [l.transform_param.crop_size for l in lrs
				if l.HasField('transform_param')]
			if len(cs):
				self.input_dim = (3, cs[0], cs[0])

	def __call__(self, clip=None, **inputs):
		from caffe import layers as L
		from collections import OrderedDict
		net = self.parsed_proto
		blobs = OrderedDict(inputs)
		for l in net.layer:
			if l.name not in inputs:
				in_place = l.top == l.bottom
				param = get_param(l)
				assert all([b in blobs for b in l.bottom]), "Some bottoms not founds: " + ', '.join([b for b in l.bottom if not b in blobs])
				tops = getattr(L, l.type)(*[blobs[b] for b in l.bottom],
				                          ntop=len(l.top), in_place=in_place,
				                          name=l.name,
				                          **param)
				if len(l.top) <= 1:
					tops = [tops]
				for i, t in enumerate(l.top):
					blobs[t] = tops[i]
			if l.name == clip:
				break
		return list(blobs.values())[-1]


import os
from pathlib import Path

def read_dataset(main_folder, format="png"):
    """
    Read an entire dataset folder using a defined set of image formats.

    Parameters
    ----------
    main_folder : str
        The full path for a folder that contains image files to be defined as
        the model training dataset
    format : str, optional
        The file format that will be used to search the images in such input
        folder. This string can be a set of formats separated by comma, e.g.

        >>> format_set = "png, jpg, tif"

        By default "png" is adopted.

        Therefore, the classes (labels) will be defined by the folder name that
        contains its image examples.

    Raises
    ------
    FileExistsError
        Folder not found with provided path

    Returns
    -------
    dict
        A structured dataset informing the numpy arrays of images that were
        indeed read (by the key `images`), the list of image classes (by the
        key `labels`) and the list of images that encounter reading errors
        (by the key `errors`)
    """
    if not os.path.exists(main_folder):
        raise FileNotFoundError("Folder not found with provided path.")

    paths = collect_filepaths(main_folder, format)
    dataset = {"data": [], "labels": [], "errors": []}

    for path in paths:
        try:
            dataset["labels"].append(__infer_label(path))
            dataset["data"].append(path)
        except (ValueError, FileExistsError):
            dataset["errors"].append(path)
    return dataset

def __infer_label(filepath):
    dir_hierarchy = str(Path(filepath).parents[0])
    label = dir_hierarchy.split(os.sep)[-1]

    return label

def collect_filepaths(main_path, format="png"):
    """
    Collect a set of files using a determined location on disk and a known list
    of file formats. This is useful to get the full path of files encountered
    on a disk folder.

    Parameters
    ----------
    main_path : str
        The full path to the main folder where the many files are found in it.
        A recursive path collection is assumed, i.e. the files are found inside
        nested folder hierarchy.
    format : str, optional
        The file format that will be used to search the images in such input
        folder. This string can be a set of formats separated by comma, e.g.

        >>> format_set = "png, jpg, tif"
        By default "png" is adopted.

    Returns
    -------
    list
        A list with file paths collect using the specific file formats.
    """
    root_path = Path(main_path)
    format_list = format.split(",")
    paths = []
    for f in format_list:
        paths.extend(root_path.rglob(f"*.{f.strip()}"))

    return [path.as_posix() for path in paths]


# https://programminghistorian.org/en/lessons/retired/intro-to-beautiful-soup
from bs4 import BeautifulSoup
import requests

url = "http://www.google.com"

response = requests.get(url)

soup = BeautifulSoup(response.text, 'html.parser')

#print nicely indented html
# print(soup.prettify())

links = soup.find_all('a')

for link in links:
    print(link.contents[0])
    print(link.get('href'))


from django.urls import path
from .views import RegisterUser, ConfirmUser, UsersLove, LoginUser, UsersView, UsersDistance

app_name = 'users'
urlpatterns = [
    path('clients/create', RegisterUser.as_view(), name='user-register'),
    path('clients/create/confirm', ConfirmUser.as_view(), name='user-register-confirm'),
    path('clients/login', LoginUser.as_view(), name='user-login'),
    path('clients/<int:id>/match', UsersLove.as_view(), name= 'user-match'),
    path('list/', UsersView.as_view({'get': 'get'})),
    path('clients/distance/<int:id1>/<int:id2>/', UsersDistance.as_view({'get': 'get'}))

]


import unittest

from os.path import abspath, basename, join, splitext as split_ext
from tempfile import mkstemp, mkdtemp
from os import remove, close, environ
from time import sleep

from itertools import combinations
try:
    from urllib.request import Request, urlopen
except ImportError:
    from urllib2 import Request, urlopen

from shutil import rmtree

class FormManagerTestCase(unittest.TestCase):
    _tmpfiles = []
    _fm_instance = None

    def setUp(self):
        environ['KIVY_FORM_DEBUG'] = '1'

    # basic class tests
    def test_singleton(self):
        from kivy.garden.formmanager import FormManager as FM

        comb = combinations(
            [FM() for i in range(3)], r=2
        )

        for a, b in comb:
            self.assertEqual(a, b)

    def test_kill(self):
        from kivy.garden.formmanager import FormManager as FM

        inst = []
        for i in range(3):
            fm = FM()
            fm.kill()
            inst.append(fm)

        for a, b in combinations(inst, r=2):
            self.assertNotEqual(a, b)

    # server tests
    def test_not_running(self):
        from kivy.garden.formmanager import FormManager
        fm = FormManager()
        self.assertFalse(fm.running)
        self.assertNotEqual(
            FormManager.get_manager(),
            None
        )

    def test_run(self):
        from kivy.garden.formmanager import FormManager
        fm = FormManager()
        self._fm_instance = fm

        # default class value until changed in run()
        self.assertEqual(fm.port, 0)

        fm.run()
        port = fm.port
        self.assertTrue(fm.running)
        self.assertTrue(port)

        fm.stop()
        self.assertFalse(fm.running)
        self.assertTrue(fm.port)
        self.assertEqual(port, fm.port)

        # remove instance
        fm.kill()

    def test_rerun(self):
        # Invalid File Descriptor -1 for socket
        # doesn't work, closed socket can't be reopen apparently
        # https://bugs.python.org/msg278691
        from kivy.garden.formmanager import FormManager
        fm = FormManager()
        self._fm_instance = fm

        # default class value until changed in run()
        self.assertEqual(fm.port, 0)

        fm.run()
        port = fm.port
        self.assertTrue(fm.running)
        self.assertTrue(port)

        fm.stop()
        self.assertFalse(fm.running)
        self.assertTrue(fm.port)
        self.assertEqual(port, fm.port)

        # assert the ValueError, because IFD -1
        # raises STDLIB selectors.py's _fileobj_to_fd
        with self.assertRaises(ValueError):
            fm.server.serve_forever()
        fm.stop()
        self.assertFalse(fm.running)
        self.assertTrue(fm.port)
        self.assertEqual(port, fm.port)

        # remove instance
        fm.kill()

    def test_dummy_post(self):
        # remove later when API is strict
        from kivy.garden.formmanager import FormManager

        fm = FormManager()
        self._fm_instance = fm
        fm.run()

        self._send_json(
            host='http://127.0.0.1',
            port=fm.port,
            data={"test": "value"}
        )

        fm.kill()

    def test_add_nonform(self):
        from kivy.garden.formmanager import FormManager, FormManagerException

        fm = FormManager()
        self._fm_instance = fm
        fm.run()

        tmpfd, tmpfn = mkstemp('.py')
        FormManagerTestCase._tmpfiles.append([tmpfd, tmpfn])
        form = object()

        with self.assertRaises(FormManagerException):
            fm.add_form(form)

        fm.kill()

    def test_add_remove_form(self):
        from kivy.garden.formmanager import FormManager, Form

        fm = FormManager()
        self._fm_instance = fm
        fm.run()

        tmpfd, tmpfn = mkstemp('.py')
        FormManagerTestCase._tmpfiles.append([tmpfd, tmpfn])
        form = Form(tmpfn)

        fm.add_form(form)
        self.assertIn(form.name, fm.forms)
        fm.remove_form(form)
        self.assertNotIn(form.name, fm.forms)

        fm.kill()

    # helper methods
    def _send_json(self, host, port, data):
        json = str(data)
        request = Request(
            host + ':' + str(port),
            bytearray(json, 'utf-8'),
            {'Content-Type': 'application/json'}
        )
        json = urlopen(request).read().decode()
        print('result:', json)

    def tearDown(self):
        # in case of assertion error, always kill the server
        if self._fm_instance:
            self._fm_instance.kill()
        environ.pop('KIVY_FORM_DEBUG')
        sleep(0.1)

class FormTestCase(unittest.TestCase):
    _tmpfiles = []
    _fm_instance = None

    form_template = (
        "from random import randint\n"
        "from kivy.config import Config\n"
        "Config.set('graphics', 'position', 'custom')\n"
        "Config.set('graphics', 'left', randint(0, 600))\n"
        "Config.set('graphics', 'top', randint(0, 600))\n"
        "from kivy.garden.formmanager import FormApp\n"
        "from kivy.lang import Builder\n"
        "from kivy.uix.boxlayout import BoxLayout\n"
        "Builder.load_string('''\n"
        "<Test>:\n"
        "    Button:\n"
        "        text: app.name\n"
        "''')\n"
        "class Test(BoxLayout):\n"
        "    pass\n"
        "class {0}(FormApp):\n"
        "    def build(self):\n"
        "        return Test()\n"
        "{0}().run()\n"
    )

    def test_name(self):
        from kivy.garden.formmanager import Form

        tmpfd, tmpfn = mkstemp('.py')
        FormTestCase._tmpfiles.append([tmpfd, tmpfn])

        form = Form(tmpfn)
        self.assertEqual(
            form.name,
            split_ext(basename(abspath(tmpfn)))[0]
        )

    def test_run_form(self):
        # needs more details
        from kivy.garden.formmanager import FormManager, Form

        fm = FormManager()
        self._fm_instance = fm
        fm.run()

        tmpdir = mkdtemp()

        tmp_form = join(tmpdir, 'form0.py')
        form_name = split_ext(basename(abspath(tmp_form)))[0]
        with open(tmp_form, 'w') as f:
            f.write(
                self.form_template.format(form_name.capitalize())
            )

        form = Form(tmp_form)
        fm.add_form(form)
        fm.run_form(form)

        # Form application is basically another Kivy app run in
        # a separate process, therefore we have to wait for it to load
        sleep(2)

        self.assertTrue(fm.forms[form.name]['active'])

        # remove form test?
        fm.kill()
        rmtree(tmpdir)

    def test_run_multiple_forms(self):
        # needs more details
        from kivy.garden.formmanager import FormManager, Form

        fm = FormManager()
        self._fm_instance = fm
        fm.run()

        tmpdir = mkdtemp()

        for i in range(3):
            tmp_form = join(tmpdir, 'form{}.py'.format(i + 1))
            form_name = split_ext(basename(abspath(tmp_form)))[0]
            with open(tmp_form, 'w') as f:
                f.write(
                    self.form_template.format(form_name.capitalize())
                )

            form = Form(tmp_form)
            fm.add_form(form)
            fm.run_form(form)

            # Form application is basically another Kivy app run in
            # a separate process, therefore we have to wait for it to load
            sleep(3)

            self.assertTrue(fm.forms[form.name]['active'])

        # remove form test?
        fm.kill()
        rmtree(tmpdir)

    def test_run_form_request_action(self):
        from kivy.garden.formmanager import FormManager, Form, FormManagerException

        fm = FormManager()
        self._fm_instance = fm
        fm.run()

        # request action on non-existing Form
        with self.assertRaises(FormManagerException):
            fm.request_action('form4', 'print', 'nope')

        self.assertEqual(fm.queue, {})

        tmpdir = mkdtemp()

        tmp_form = join(tmpdir, 'form4.py')
        form_name = split_ext(basename(abspath(tmp_form)))[0]
        with open(tmp_form, 'w') as f:
            f.write(
                self.form_template.format(form_name.capitalize())
            )

        form = Form(tmp_form)
        fm.add_form(form)
        fm.run_form(form)

        # Form application is basically another Kivy app run in
        # a separate process, therefore we have to wait for it to load
        sleep(2)

        self.assertTrue(fm.forms[form.name]['active'])

        # request action for Form1
        fm.request_action('form4', 'print', 'test')
        self.assertEqual(
            fm.queue,
            {'form4': [['print', 'test']]}
        )

        sleep(1)

        # after request the action is popped,
        # but Form remains in the queue as a key
        self.assertEqual(fm.queue, {"form4": []})

        # after the Form is removed, the key should too
        fm.remove_form(form)
        self.assertNotIn(form.name, fm.forms)
        self.assertEqual(fm.queue, {})

        fm.kill()
        rmtree(tmpdir)

    def test_run_form_request_call(self):
        from kivy.garden.formmanager import FormManager, Form, FormManagerException

        fm = FormManager()
        self._fm_instance = fm
        fm.run()
        self.assertEqual(fm.queue, {})

        tmpdir = mkdtemp()
        tmp_form = join(tmpdir, 'form5.py')
        form_name = split_ext(basename(abspath(tmp_form)))[0]
        with open(tmp_form, 'w') as f:
            f.write(
                self.form_template.format(form_name.capitalize())
            )

        form = Form(tmp_form)
        fm.add_form(form)
        fm.run_form(form)

        # Form application is basically another Kivy app run in
        # a separate process, therefore we have to wait for it to load
        sleep(2)

        self.assertTrue(fm.forms[form.name]['active'])

        # request action for Form1
        fm.request_action('form5', 'call', ['self', 'open_settings'])
        self.assertEqual(
            fm.queue,
            {'form5': [['call', ['self', 'open_settings']]]}
        )

        sleep(1)

        # after request the action is popped,
        # but Form remains in the queue as a key
        self.assertEqual(fm.queue, {"form5": []})

        # after the Form is removed, the key should too
        fm.remove_form(form)
        self.assertNotIn(form.name, fm.forms)
        self.assertEqual(fm.queue, {})

        fm.kill()
        rmtree(tmpdir)

    def tearDown(self):
        # in case of assertion error, always kill the server
        if self._fm_instance:
            self._fm_instance.kill()
        sleep(1)

def tearDownModule():
    # throw away all temporary files after testing
    # therefore nothing should use the files here
    for desc, tmp in FormManagerTestCase._tmpfiles:
        close(desc)
        remove(tmp)
    for desc, tmp in FormTestCase._tmpfiles:
        close(desc)
        remove(tmp)

if __name__ == '__main__':
    unittest.main()

{'FM': 'kivy.garden.formmanager.FormManager', 'FormManager': 'kivy.garden.formmanager.FormManager', 'FormManagerException': 'kivy.garden.formmanager.FormManagerException'}._tmpfiles.append

{'FM': 'kivy.garden.formmanager.FormManager', 'FormManager': 'kivy.garden.formmanager.FormManager', 'FormManagerException': 'kivy.garden.formmanager.FormManagerException', 'Form': 'kivy.garden.formmanager.Form'}._tmpfiles.append

{'Form': 'kivy.garden.formmanager.Form', 'FormManager': 'kivy.garden.formmanager.FormManager', 'FormManagerException': 'kivy.garden.formmanager.FormManagerException'}._tmpfiles


import re
from collections import defaultdict
import random
import operator
from math import ceil

from reportlab import xrange

from statistics.syllable_per_word import get_mean_syllable
from utils.spanish_syllables import how_many_syllables_sent, how_many_syllables_word
from .grammar import Grammar
from .symbol import Symbol

class GrammarSensitiveGenerator(object):

    def __init__(self, model, grammar, tokens):
        """
        :type tokens: set of string to Symbol
        :type grammar: object of model.grammar.Grammar
        :type model: object of model.ngram.LanguageModel
        """
        self._n = model._n
        self.model = model
        self.grammar = grammar
        self.tokens = tokens
        # compute the probabilities
        probs = defaultdict(dict)
        count = model._count.copy()
        count.pop(())
        count.pop(('<s>',)) # no queremos que exista prob de empezar una nueva oracion
        for gram in count:
            gram = list(gram)
            last_word = gram.pop()
            if last_word in tokens.keys():
                probs[tokens[last_word]].setdefault(tuple(gram), {}).update({last_word : model.cond_prob(last_word, gram)})

        self._probs = dict(probs)

        #        for key, value in self._probs.items():
        #            print(sum(value.values()))
                # sort in descending order for efficient sampling
        self._sorted_probs = defaultdict(dict)
        for tok in self._probs.keys():
            for key, value in self._probs[tok].items():
                self._sorted_probs[tok].setdefault(key, {})
                self._sorted_probs[tok][key] = sorted(value.items(), key=operator.itemgetter(1), reverse=True)

    def generate_sent(self):
        """Randomly generate a sentence.
        returns string
        """
        n = self._n
        s = self.grammar.get_start()
        grammar = self.grammar
        word = "<s>"
        sentence = [word]
        alpha = [s]
        product = [1, 2, 3]
        i = 0
        while len(product) > 1:
            product = random.choice(grammar.get_possible_productions(alpha))\
                .produce()
            word = self.generate_word(tuple(sentence[i:i + n - 1]), product[0])
            # get rid of the old non terminal, and replace it by the new product
            alpha = alpha[:-1] + product
            sentence.append(word)
            i += 1
        return ' '.join(sentence[1:])

    '''
    # Brute forcing conditions, it may be impossible or highly improbable, should be time catched
    def generate_sent_restricted(self, conditions):
        """
        Randomly generate a sentence of given syllables that matches array of conditions
                In case is not possible, returns -1
        :type conditions: List of RegEx
        """
        sent = self.generate_sent()
        while not conditions.are_all_attained(sent):
            print(sent)
            sent = self.generate_sent()
        return sent
    '''

    def get_word_that_matches(self, regex):
        r = random.choice(tuple(self.model.vocabulary()))
        i = 0
        while re.match(regex, r, re.IGNORECASE) is None and i < self.model.V():
            r = random.choice(tuple(self.model.vocabulary()))
            i += 1
        return r

    def generate_verse(self, syllables, rhyme=".*", attempts=10, persistence=10):
        """
        Generate a verse of given syllables and rhyme
        :param attempts: how many times will we try to rhyme
        :param syllables: Amount of syllables
        :param rhyme: regex for last word
        :param persistence: how many times it will try to find a verse matching both rhyme and syllable
        :return: string
        """
        n = self._n
        s = self.grammar.get_start()
        grammar = self.grammar
        # as many attempts as we are allowed (attempts as words that rhyme that we will try to put in a sentence with
        # given amount of syllables
        for _ in xrange(attempts):
            # get me any word that rhymes and it will be the last
            word = self.get_word_that_matches(rhyme)
            # what's its token?
            token = self.tokens[word]
            # estimate the amount of words we will need to match the required syllables
            w = ceil(syllables/get_mean_syllable(self.model))
            # around that much of words we will generate (it's clumsy)
            # we need to do this to choose how deep into our grammar to look for the token we have, the token can be
            # anywhere, it's useless to look for it in depth 1 given that the words has less than x syllables
            #### in case of i=4 we will try sentences of lenght 3, 4, 5 and 6
            for i in range(w-1, w+2):
                # we get a list of all the possible productions that get will us to 'word''s token, given it's context
                # [token] means we are looking for producctions that return only token, and no NonTerminals
                # grammar.non_terminals_hierarchy[i] comes as "the nonterminal for i depth"
                possibilities = grammar.get_possible_produced([token], grammar.non_terminals_hierarchy[i])
                if not possibilities:
                    # if it's empty then we have a token that can't be the last one
                    break
                # any of those productions, we pick it's context alpha, as we ignore beta in this program. [:-1] because
                # we already have that one
                sentence_tokens = random.choice(possibilities).alpha[:-1]
                patience = persistence
                # let's match token to word, attempting to satisfy the syllables condition.
                # until we lose our patience as a poet
                while patience > 0:
                    sentence = ["<s>"]
                    for token in sentence_tokens:
                        sentence.append(self.generate_word(tuple(sentence[i:i + n - 1]), token))
                    sentence.append(word)
                    # does it comply with the requested sentences?
                    if how_many_syllables_sent(' '.join(sentence[1:])) != syllables:
                        patience -= 1
                    else:
                        return ' '.join(sentence[1:])
        return -1

    def generate_word_restricted(self, regex, prev_tokens=None, token=None):
        """Randomly generate a token, given prev_tokens.

        prev_tokens -- the previous n-1 tokens (optional only if n = 1).
        returns -1 if it's impossible to match the regex
        """
        assert isinstance(token, Symbol)
        if token == Symbol('ε', True):
            return "</s>"
        x = random.choice(list(self._sorted_probs[token.symbol].keys()))
        if prev_tokens not in self._sorted_probs[token.symbol].keys():
            options = self._sorted_probs[token.symbol][x]
        else:
            options = self._sorted_probs[token.symbol][prev_tokens]
        for option in options:
            if re.match(regex, option[0]) is not None:
                return option[0]
        return -1

    def generate_word(self, prev_tokens=None, token=None):
        """Randomly generate a token, given prev_tokens.

        prev_tokens -- the previous n-1 tokens (optional only if n = 1).
        """
        assert isinstance(token, Symbol)
        if token == Symbol('ε', True):
            return "</s>"
        r = random.random()
        i = 0
        x = random.choice(list(self._sorted_probs[token].keys()))
        if prev_tokens not in self._sorted_probs[token].keys():
            word, prob = self._sorted_probs[token][x][0]
            l = len(self._sorted_probs[token][x]) # Para que addone no se pase
        else:
            word, prob = self._sorted_probs[token][prev_tokens][0]
            l = len(self._sorted_probs[token][prev_tokens])  # Para que addone no se pase
        acum = prob
        '''
        sea una acumulada discreta escalonada tenemos algo asi
         [0] [1]         [2]        r              [3]    [4][5]             [6]     [7]   [8]
        0|----|-----------|-------------------------|------|--|---------------|-------|-----|1
        por eso hay mas chances de que r caiga en la posicion de mayor probabilidad
        y por eso lo ordenamos
        '''
        while r > acum:  # Buscamos en la funcion acumulada nuestro valor random
            i += 1
            i = i % l
            if prev_tokens not in self._sorted_probs[token].keys():
                word, prob = self._sorted_probs[token][x][i]
            else:
                word, prob = self._sorted_probs[token][prev_tokens][i]
            acum += prob
        return word


#!/usr/bin/env python3

import datetime
import code
import serial
import asyncio
import binascii
import struct
import time
import logging
import warnings

from commands import Commands

if not logging.getLogger().handlers:
    logging.basicConfig(level=logging.DEBUG)

log = logging.getLogger(__name__)

def warn_untested(fn):
    def _warn_wrap(*args, **kwargs):
        warnings.warn(f"{fn.__name__} untested!")
        fn(*args, **kwargs)
    return _warn_wrap

def warn_seems_broken(fn):
    def _warn_wrap(*args, **kwargs):
        warnings.warn(f"{fn.__name__} seems broken...")
        fn(*args, **kwargs)
    return _warn_wrap

class GM:

    def __init__(self, port):
        self.serial = serial.Serial(port=port,
                                    baudrate=57600,
                                    timeout=1.0)

    def get_serial(self):
        self.serial.write(Commands.get_serial)
        serblob = self.serial.read(7)
        return binascii.hexlify(serblob).decode('ascii')

    def get_version(self):
        self.serial.write(Commands.get_version)
        return self.serial.read(14).decode('ascii')

    def get_voltage(self):
        self.serial.write(Commands.get_voltage)
        return ord(self.serial.read(1)) / 10.0

    @staticmethod
    def cp_calc(data):
        return struct.unpack("h", data)[0] & 0x3f00

    def get_cpm(self):
        self.serial.write(Commands.get_cpm)
        return self.read_cps()

    def get_cps(self):
        self.serial.write(Commands.get_cps)
        return self.read_cps()

    def auto_cps_on(self):
        self.serial.write(Commands.auto_cps_on)

    def auto_cps_off(self):
        self.serial.write(Commands.auto_cps_off)

    def read_cps(self):
        return self.cp_calc(self.serial.read(2))

    def get_datetime(self):
        self.serial.write(Commands.get_datetime)
        data = self.serial.read(7)
        print(len(data), binascii.hexlify(data))
        # assert data[6] == 0xAA
        return datetime.datetime((2000+data[0]), *data[1:5])

    def power_off(self):
        self.serial.write(Commands.power_off)

    def power_on(self):
        self.serial.write(Commands.power_on)

    def reboot(self):
        self.serial.write(Commands.reboot)

    def test(self):
        print(f"serial: {self.get_serial()}")
        print(f"version: {self.get_version()}")
        print(f"voltage: {self.get_voltage()}V")
        print(f"cpm: {self.get_cpm()}")
        print(f"cps: {self.get_cps()}")
        print(f"datetime: {self.get_datetime()}")

    def test_autocps(self):
        with self.auto_cps(count=5) as cps:
            for reading in cps:
                print(f"autocps: {reading}")

    class AutoCPS:
        def __init__(self, device, count=None):
            self.device = device
            self.remaining = count

        def __iter__(self):
            return self

        def __enter__(self):
            self.device.auto_cps_on()
            return self

        def __exit__(self, typ, value, tb):
            self.device.auto_cps_off()

        def __next__(self):
            if self.remaining is not None:
                if not self.remaining:
                    raise StopIteration()
                self.remaining -= 1
            return self.device.read_cps()

    def auto_cps(self, count=None):
        return self.AutoCPS(self, count=count)

    FULL_USER_SIZE = 65536
    PAGE_SIZE = 2048
    @warn_seems_broken
    def fetch_page(self, num, size=PAGE_SIZE):
        offset = num * size
        offset_b = struct.pack(">I", offset)[1:]
        # bug: request A-1 when want A
        size_b = struct.pack(">H", size - 1)
        cmd = Commands.build_userdata_read(offset_b, size_b)
        if self.serial.in_waiting:
            self.serial.reset_input_buffer()
        self.serial.write(cmd)
        return self.serial.read(size)

    @warn_seems_broken
    def fetch_pages(self, num=None, size=None):
        if size is None:
            size = self.PAGE_SIZE
        if num is None:
            num = self.FULL_USER_SIZE // size
        ret = b""
        for n in range(num):
            while True:
                b = self.fetch_page(n, size)
                print(n, len(b), len(b) - size)
                if len(b) < size:
                    self.serial.reset_input_buffer()
                    self.serial.reset_output_buffer()
                else:
                    break
            ret += b
        self.serial.reset_input_buffer()
        self.serial.reset_output_buffer()
        return ret

    @warn_seems_broken
    def gen_history_records(self, data=None):
        if data is None:
            data = self.fetch_pages()
        # remove after-data bits
        ham = data.rstrip(b"\xff")
        records = ham.split(b"\x55\xaa")
        # now, each record is:
        # 0x00 date
        # 0x01 data

        basedate = None
        interval = None
        duration = None
        for record in records:
            print(len(record), binascii.hexlify(record))
            if not record:
                continue
            elif record[0] == b"\x00" and not basedate:
                basedate = datetime.datetime(
                    # HAVE WE LEARNED NOTHING!?
                    year=(2000 + record[1]),
                    month=record[2],
                    day=record[3],
                    hour=record[4],
                    minute=record[5],
                    second=record[6]
                )
            elif basedate:
                # fuckin' math(s)
                # 1 - per second
                # 60 - per minute
                # 3600 - per hour
                interval = 60 ** (record[0] - 1)
                duration = datetime.timedelta(seconds=interval)
                for i in range(1, len(record), 2):
                    sample = record[i:i+2]
                    cpx = self.cp_calc(sample)
                    time_at = basedate + (i//2) * duration
                    yield cpx, time_at
            else:
                print(f"??? {record}")

    @warn_untested
    def get_cfg(self):
        self.serial.write(Commands.get_cfg)
        return self.serial.read(256)

# hint: device = GM('/dev/ttyUSB0')
#       device.test()

code.interact(local=dict(globals(), **locals()))


#coding=utf-8
import urllib2
from bs4 import BeautifulSoup
import time
import user_agents
import random
import os
import sys

def get_htmlsoup(site):

    site_page = url_open(site)
    soup = BeautifulSoup(site_page, 'html.parser')
    return soup
#获取经过beautifusoup处理过后的结果
def url_open(site):
    randomarry = random.choice(user_agents.user_agent_list)
#随机挑选一个user_agent文件头
    headers = {
    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.6',
    'X-Requested-With':'XMLHttpRequest',
    'User-Agent':randomarry
    }
#手动添加完整一套文件头，假装是不同的浏览器进行访问
    data = None

    requests = urllib2.Request(site,data,headers)
    response = urllib2.urlopen(requests)
    site_page = response.read()
    return site_page

def is_girl(url):
    content_soup = get_htmlsoup(url)
    author = content_soup.find(id="articleAuthor")
    author_url = author['href']
    #点进每一条的作者页面查看性别
    author_page = get_htmlsoup(author_url)
    sex_type = author_page.find(class_="profile-info-detail")
    if 'class="icon-male"' in str(sex_type):
        return 0

def refresh(count_list,list_array,filename):
    tmp_list = count_list[list_array:]
    config=open(str(filename)+'.txt','w')
    config.write('')
    config=open(str(filename)+'.txt','a')
    for i in tmp_list:
        config.write(i)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import django.utils.timezone
import app.models

class Migration(migrations.Migration):

    dependencies = [
        ('app', '0030_userprofile_admin'),
    ]

    operations = [
        migrations.CreateModel(
            name='Wallet',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('total', models.FloatField(default=0, null=True, verbose_name=b'Wallet', blank=True)),
                ('code', models.CharField(default=app.models.random_string, max_length=100)),
                ('create_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)),
                ('update_at', models.DateTimeField(auto_now=True)),
                ('slug', models.SlugField(editable=False)),
                ('user', models.ForeignKey(related_name='wallet_owner', verbose_name=b'Usuario', blank=True, to='app.UserProfile', null=True)),
            ],
            options={
                'ordering': ['create_at'],
                'verbose_name': 'Orden de compra',
                'verbose_name_plural': 'Ordenes de compra',
            },
        ),
    ]


import matplotlib.pyplot as plt
import statsmodels.api as sm
import numbers
import pandas as pd
from tabulate import tabulate
from statsmodels.stats.outliers_influence import summary_table
from typing import Tuple, Dict
import numpy as np

def print_tabulate(df: pd.DataFrame):
    print(tabulate(df, headers=df.columns, tablefmt="orgtbl"))

def transform_variable(df: pd.DataFrame, x:str)->pd.Series:
    if isinstance(df[x][df.index[0]], numbers.Number):
        return df[x] # type: pd.Series
    else:
        return pd.Series([i for i in range(0, len(df[x]))])

def linear_regression(df: pd.DataFrame, x:str, y: str)->Dict[str, float]:
    fixed_x = transform_variable(df, x)
    model= sm.OLS(list(df[y]),sm.add_constant(fixed_x), alpha=0.05).fit()
    bands = pd.read_html(model.summary().tables[1].as_html(),header=0,index_col=0)[0]
    #print_tabulate(pd.read_html(model.summary().tables[1].as_html(),header=0,index_col=0)[0])
    coef = pd.read_html(model.summary().tables[1].as_html(),header=0,index_col=0)[0]['coef']
    r_2_t = pd.read_html(model.summary().tables[0].as_html(),header=None,index_col=None)[0]
    return {'m': coef.values[1], 'b': coef.values[0], 'r2': r_2_t.values[0][3], 'r2_adj': r_2_t.values[1][3], 'low_band': bands['[0.025'][0], 'hi_band': bands['0.975]'][0]}

def plt_lr(df: pd.DataFrame, x:str, y: str, m: float, b: float, r2: float, r2_adj: float, low_band: float, hi_band: float, colors: Tuple[str,str]):
    fixed_x = transform_variable(df, x)
    plt.plot(df[x],[ m * x + b for _, x in fixed_x.items()], color=colors[0])
    plt.fill_between(df[x],
                     [ m * x  + low_band for _, x in fixed_x.items()],
                     [ m * x + hi_band for _, x in fixed_x.items()], alpha=0.2, color=colors[1])

def normalize_distribution(dist: np.array, n: int)->np.array:
    b = dist - min(dist) + 0.000001
    c = (b / np.sum(b)) * n
    return np.round(c)

def yearAverage(reviews):
    print('hola mundo')
# begin_date = '2016-01-01'
# end_date = '2022-01-01'
# date_range = pd.date_range(start=begin_date, end=end_date, freq='1D')
# norm_dist = np.random.standard_normal(len(date_range))
# sales = normalize_distribution(norm_dist, 50*len(date_range))
# df = pd.DataFrame({'yearpublished': date_range, 'avgYear': sales}) # type: pd.DataFrame
# df.to_csv('csv/sales.csv', index=False)

reviews = pd.read_csv('csv/reviewsYear.csv')
reviews = reviews.groupby("yearpublished")\
    .aggregate(avgYear=pd.NamedAgg(column="average", aggfunc=pd.DataFrame.mean))
reviews.to_csv('csv/avgYear.csv' ,index=True)

full_df = pd.read_csv('csv/avgYear.csv')
print(f"full -> mean: {np.mean(full_df['avgYear'])}, sd: {np.std(full_df['avgYear'])}")
df = full_df.tail(100)
x = "yearpublished"
y= "avgYear"
# full_df.plot(x=x,y=y, kind='scatter')
# plt.xticks(rotation=90)
# plt.savefig('img/full_avgYear_yearpublished_m.png')
# plt.close()

df.plot(x=x,y=y, kind='scatter')
a = linear_regression(df, x,y)
plt_lr(df=df, x=x, y=y, colors=('red', 'orange'), **a)
a = linear_regression(df.tail(75), x,y)
plt_lr(df=df.tail(75), x=x, y=y, colors=('blue', 'blue'), **a)
a = linear_regression(df.tail(50), x,y)
plt_lr(df=df.tail(50), x=x, y=y, colors=('green', 'green'), **a)
#df_j = df[pd.to_datetime(df[x]).dt.dayofweek == 4]
#print_tabulate(df_j)
#a = linear_regression(df_j, x,y)
#plt_lr(df=df_j, x=x, y=y, colors=('blue', 'blue'), **a)
#
plt.xticks(rotation=90)
plt.savefig('img/lr_avgYear_yearpublished_m.png')
plt.close()

"""df2 = full_df.loc[(pd.to_datetime(full_df[x])>='2019-11-11') & (pd.to_datetime(full_df[x]) < '2020-01-02')]
dfs = [
    ('50D', df),
    ('10D', df.tail(10)),
    ('5D', df.tail(5)),
    ('jueves', df[pd.to_datetime(df[x]).dt.dayofweek == 4]),
    ('50D-1Y', df2),
    ('10D-Y', df2.tail(10)),
    ('5D-Y', df2.tail(5)),
    ('jueves-Y', df2[pd.to_datetime(df2[x]).dt.dayofweek == 4]),
]
lrs = [(title, linear_regression(_df,x=x,y=y), len(_df)) for title, _df in dfs]
lrs_p = [(title, lr_dict["m"]*size  + lr_dict["b"], lr_dict) for title, lr_dict, size in lrs]
print(lrs_p)"""