"""

This module contains base classes and useful functions for the retina module.

"""

import litus
import theano
import theano.tensor as T
import numpy as np
import matplotlib.pylab as plt
from theano.tensor.nnet.conv3d2d import conv3d
from theano.tensor.signal.conv import conv2d
import uuid

dtensor5 = T.TensorType('float64', (False,)*5)

A = dtensor5('A')
B = dtensor5('B')
C = conv3d(A,B)
_conv_func = theano.function(inputs=[A,B], outputs=C)

def conv(a,b,padding_things_equal=[1,3,4],padding_things_tail=[1],*args,**kwargs):
    a_ = a.copy()
    b_ = b.copy()
    a_ = np.pad(a_,[(s-1,s-1) for si,s in enumerate(b_.shape)],mode='constant')
    return _conv_func(a_,b_)

### Functions to create filters
#

def minimize_filter(f,filter_epsilon = 0.0, minimize_xy=True, minimize_t_tail=True, minimize_t_start=False):
    """
        reduces a filter by taking of the sides if they are smaller than :py:obj:`filter_epsilon`
    """
    if np.max(np.abs(f)) <= filter_epsilon:
        return np.array([0]).reshape((1,1,1,1,1))
    if minimize_xy:
        if f.shape[3] > 1:
            while np.sum(np.abs(f[:,:,:,0,:])) + np.sum(np.abs(f[:,:,:,-1,:])) <= filter_epsilon*2.0:
              f = f[:,:,:,1:-1,:]
        if f.shape[4] > 1:
            while np.sum(np.abs(f[:,:,:,:,0])) + np.sum(np.abs(f[:,:,:,:,-1])) <= filter_epsilon*2.0:
                f = f[:,:,:,:,1:-1]
    if f.shape[1] > 1:
        if minimize_t_start:
            while np.sum(np.abs(f[:,0,:,:])) <= filter_epsilon:
                f = f[:,1:,:,:,:]
        if minimize_t_tail:
            while np.sum(np.abs(f[:,-1,:,:])) <= filter_epsilon:
                f = f[:,:-1,:,:,:]
    return f.copy()

def m_t_filter(steepness=2.0,relative_weight=1.0,normalize=True,minimize=True,retina=None,epsilon=0.0001):
    """
        A T filter creates a transient response by combining a sharp positive (1) term with a negative exponential that has area :py:obj:`relative_weight`.
        The sum of the area of this filter is 1-:py:obj:`relative_weight`.
    """
    if retina is not None:
        steepness = retina.seconds_to_steps(steepness)
    length = 5.0*steepness
    T_filter = -1*np.exp(-np.arange(0.0,length)/float(steepness))
    if normalize:
        T_filter = -1*T_filter/np.sum(T_filter)
    T_filter = relative_weight * T_filter
    T_filter = np.array([1] + T_filter.tolist())
    if minimize:
        return minimize_filter(T_filter.reshape((1,len(T_filter),1,1,1)),epsilon)
    return T_filter.reshape((1,len(T_filter),1,1,1))

def m_e_filter(steepness=2.0,normalize=True,minimize=True,retina=None,epsilon=0.001):
    """
        An exponential filter.
    """
    if retina is not None:
        steepness = retina.seconds_to_steps(steepness)
    length = 10.0*steepness
    kernel = np.exp(-np.arange(0.0,length)/float(steepness))
    if normalize:
        kernel = kernel/np.sum(kernel)
    kernel = kernel.reshape((1,len(kernel),1,1,1))
    if minimize:
        kernel = minimize_filter(kernel,epsilon)
    return kernel

def m_en_filter(n,tau=2.0,normalize=True,minimize=True,retina=None,epsilon=0.001):
    """
        Creates an exponential cascade filter that is convolved :py:obj:`n` times.

        :py:obj:`normalize` does nothing as the kernels are already generated normalized.
    """
    if n == 0:
        return m_e_filter(tau,normalize=normalize,minimize=minimize,retina=retina,epsilon=epsilon)
    if retina is not None:
        tau = retina.seconds_to_steps(tau)
    length = int(4.0*tau)
    t = np.linspace(0.0,length,length)
    kernel = (n*t)**n * np.exp(-n*t/tau) / (np.math.factorial(n-1) * tau**(n+1))
    if normalize:
        return kernel.reshape((1,len(kernel),1,1,1))/np.sum(kernel)
    if minimize:
        return minimize_filter(kernel.reshape((1,len(kernel),1,1,1)),epsilon)
    return kernel.reshape((1,len(kernel),1,1,1))

def m_g_filter(x_sig,y_sig,normalize=False,retina=None,minimize=False,epsilon=0.00001, even=None, make_uneven=False):
    """
        A 2d gaussian in a 5d data structure (1 x time x 1 x X x Y)

        x_sig and y_sig are the standard deviations in x and y direction.

        if :py:obj:`even` is not None, the kernel will be either made to have even or uneven side lengths, depending on the truth value of :py:obj:`even`.
    """
    if retina is not None:
        x_sig = retina.degree_to_pixel(x_sig)
        y_sig = retina.degree_to_pixel(y_sig)
    x = np.ceil(x_sig*5.0)+1.0
    y = np.ceil(y_sig*5.0)+1.0
    if x_sig == 0 or y_sig == 0:
        return np.array([1.0]).reshape((1,1,1,1,1))
    if even is not None:
        if even:
            if x%2 == 1:
                x += 1
            if y%2 == 1:
                y += 1
        if not even:
            if x%2 == 0:
                x += 1
            if y%2 == 0:
                y += 1
    x_gauss = np.exp(-(1.0/float(x_sig)**2)*(np.arange(-np.floor(x/2.0)+(0.5 if even else 0.0),np.ceil(x/2.0)+(0.5 if even else 0.0)))**2)
    y_gauss = np.exp(-(1.0/float(y_sig)**2)*(np.arange(-np.floor(y/2.0)+(0.5 if even else 0.0),np.ceil(y/2.0)+(0.5 if even else 0.0)))**2)
    kernel = np.prod(np.meshgrid(x_gauss,y_gauss),0).reshape((1,1,1,len(y_gauss),len(x_gauss)))
    if normalize:
        kernel = kernel/np.sum(kernel)
    if minimize:
        kernel = minimize_filter(kernel,epsilon)
    return kernel

def m_g_filter_2d(x_sig,y_sig,normalize=False,retina=None,minimize=False,epsilon=0.00001, even=None, make_uneven=False):
    """
        A 2d gaussian.

        x_sig and y_sig are the standard deviations in x and y direction.

        if :py:obj:`even` is not None, the kernel will be either made to have even or uneven side lengths, depending on the truth value of :py:obj:`even`.
    """
    if retina is not None:
        x_sig = retina.degree_to_pixel(x_sig)
        y_sig = retina.degree_to_pixel(y_sig)
    x = x_sig*5.0
    y = y_sig*5.0
    if x_sig == 0 or y_sig == 0:
        return np.array([1]).reshape((1,1))
    if even is not None:
        if even:
            if x%2 == 1:
                x += 1
            if y%2 == 1:
                y += 1
        if not even:
            if x%2 == 0:
                x += 1
            if y%2 == 0:
                y += 1
    x_gauss = np.exp(-(1.0/float(x_sig)**2)*(np.arange(-np.floor(x/2.0)+(0.5 if even else 0.0),np.ceil(x/2.0)+(0.5 if even else 0.0)))**2)
    y_gauss = np.exp(-(1.0/float(y_sig)**2)*(np.arange(-np.floor(y/2.0)+(0.5 if even else 0.0),np.ceil(y/2.0)+(0.5 if even else 0.0)))**2)
    kernel = np.prod(np.meshgrid(x_gauss,y_gauss),0)
    if normalize:
        kernel = kernel/np.sum(kernel)
    if minimize:
        kernel = minimize_filter(kernel,epsilon)
    return kernel

def fake_filter(*actual_filters,**kwargs):
    """
        creates a fake filter that contains a single 1. The shape is the shape of all argument filters combined.

        This is needed if the shape of a term has to be adjusted to another term that had originally the same shape, but then got filtered by multiple filters.
    """
    shapes = np.array(np.sum([[aa-1 for aa in a.shape] for a in actual_filters],0) + np.ones(5),dtype=int)
    new_filter = np.zeros(shapes)
    if kwargs.get("centered",True):
        #print len(new_filter)/2
        new_filter[:,0,:,new_filter.shape[-2]/2,new_filter.shape[-1]/2] = 1
        #new_filter[-1] = 1
    else:
        #new_filter[0] = 1
        new_filter[:,0,:,0,0] = 1
    return new_filter.reshape(shapes)
def _depr_fake_filter(*actual_filters):
    shapes = np.prod([a.shape for a in actual_filters],0)
    new_filter = np.zeros(shapes).flatten()
    new_filter[0] = 1
    return new_filter.reshape(shapes)

def fake_filter_shape(*actual_filters):
    """
        returns the shape of all argument filters combined.
    """
    shapes = np.array(np.sum([[aa-1 for aa in a.shape] for a in actual_filters],0) + np.ones(5),dtype=int)
    return shapes

def find_nonconflict(name,names):
    if name in names:
        i = 0
        while name+' '+str(i) in names:
            i+=1
        names.append(name+' '+str(i))
        return name
    else:
        names.append(name)
        return name

def ab_filter_exp(tau,step = 0.001):
    """ create an Exp filter and return arrays for the coefficients

    TODO: describe how to use a and b

    """
    if tau < 0:
        raise Exception("Negative time constants are not implemented")
    if tau == 0:
        a = np.array([1.0])
        b = np.array([])
        return [a,b]
    a = np.array([1.0,-np.exp(-step/tau)])
    b = np.array([1.0-np.exp(-step/tau)])
    return [a,b]

def ab_filter_exp_cascade(self,tau, n, step = 0.001):
    """ create an ExpCascade filter and return arrays for the coefficients """
    from scipy.misc import comb
    tau = float(tau)
    n = int(n)
    if tau < 0:
        raise Exception("Negative time constants are not implemented")
    if n < 0:
        raise Exception("Negative cascade number is not allowed")
    if tau == 0:
        a = np.array([1.0])
        b = np.array([])
        return [a,b]
    tauC = tau/float(n) if n > 0 else tau
    c = np.exp(-step/tauC)
    N = n + 1
    a = np.array([(-c)**(i)*comb(N,i) for i in range(N+1)])
    b = np.array([(1.0-c)**N])
    return [a,b]

def concatenate_time(a,b):
    """
        concatenates ndarrays on the 'typical' time axis used in this implementation:

            If the objects are 1d, they are concatenated along this axis.
            (2d are static images only)
            If the data is 3d, then the first axis is time.
            If the data is 4d, then it is assumed that it is a collection of 3d objects, thus time is at the second index.
            5d Objects are usually used as filters, while the first and third index are unused. Time is the second dimension.
            6d objects are assumed to be collections of 5d objects such that time is at the third index.

        If the data is a list or a tuple, elements from both a and b are zipped and concatenated along time.

    """
    if type(a) in [list,tuple] and type(b) in [list,tuple]:
        return [concatenate_time(a_,b_) for (a_,b_) in zip(a,b)]
    if len(a.shape) == 1 and len(b.shape) == 1:
        return np.concatenate([a,b],0)
    if len(a.shape) == 3 and len(b.shape) == 3:
        return np.concatenate([a,b],0)
    if len(a.shape) == 4 and len(b.shape) == 4:
        return np.concatenate([a,b],1)
    if len(a.shape) == 5 and len(b.shape) == 5:
        return np.concatenate([a,b],1)
    if len(a.shape) == 6 and len(b.shape) == 6:
        return np.concatenate([a,b],2)

def deriche_coefficients(density):
    """
       Creates deriche coefficients for a given map of filter density
    """
    alpha = 1.695 * density
    ema = np.exp(-alpha)
    ek = (1-ema)*(1-ema) / (1+2*alpha*ema - ema*ema)
    A1 = ek
    A2 = ek * ema * (alpha-1.0)
    A3 = ek * ema * (alpha+1.0)
    A4 = -ek*ema*ema
    B1 = 2*ema
    B2 = -ema*ema
    return {'A1':A1, 'A2':A2, 'A3':A3, 'A4':A4, 'B1':B1, 'B2':B2 }


from constants import *
from dataset import pad_sequences

from utils import Timer
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split

import numpy as np
import tensorflow as tf
import os
import shutil

if not os.path.exists(MODEL_PATH):
    os.makedirs(MODEL_PATH)

shutil.copy2('constants.py', MODEL_PATH)

seed = 13
np.random.seed(seed)

class Pointwise:
    def __init__(self, embeddings, data):
        self.embeddings = embeddings
        self.data = data

        self.word_embedding_dim = WORD_EMBEDDING_DIM
        self.filter_sizes = FILTER_SIZES
        self.num_filters = NUM_FILTERS
        self.dropout_keep_prob = DROPOUT_KEEP_PROB
        self.l2_reg_lambda = L2_REG_LAMBDA
        self.learning_rate = LEARNING_RATE
        self.max_input_word = MAX_INPUT_WORD
        self.hidden_num = HIDDEN_NUM

        self.batch_size = BATCH_SIZE
        self.trainable = TRAINABLE
        self.num_epochs = NUM_EPOCHS
        self.is_early_stopping = IS_EARLY_STOPPING

        self.para = []

    def load_data(self):
        timer = Timer()
        timer.start("Loading start")

        self.data = self.data.data
        all_idx = list(self.data.keys())

        if self.is_early_stopping:
            self.train_idx, self.dev_idx = train_test_split(all_idx, test_size=0.3, random_state=seed)
            print("Number of validating mentions: {}".format(len(self.dev_idx)))
        else:
            self.train_idx = all_idx
        print("Number of training mentions: {}".format(len(self.train_idx)))

        timer.stop()

    def build(self):
        self._add_placeholder()
        self._add_model_op()
        self._add_train_op()

    def _add_placeholder(self):
        self.m_word_ids = tf.placeholder(tf.int32, [None, self.max_input_word], name="m_word_ids")
        self.n_word_ids = tf.placeholder(tf.int32, [None, self.max_input_word], name="n_word_ids")

        self.y = tf.placeholder(tf.int32, [None, 2], name="input_y")
        self.idx = tf.placeholder(tf.int32, [None], name="idx")

        self.dropout_keep_op = tf.placeholder(tf.float32, name="dropout_keep_prob")

    def _add_model_op(self):
        # embeddings
        with tf.name_scope('embeddings'):
            _word_embeddings = tf.Variable(self.embeddings, name="word_embeddings", dtype=tf.float32, trainable=TRAINABLE)
            self.m_word_embedding = self._get_embedding(_word_embeddings, self.m_word_ids)
            self.n_word_embedding = self._get_embedding(_word_embeddings, self.n_word_ids)

        # convolution
        self.kernels = []
        for i, filter_size in enumerate(self.filter_sizes):
            with tf.name_scope('conv-pool-%s' % filter_size):
                filter_shape = [filter_size, self.word_embedding_dim, 1, self.num_filters]
                W = tf.get_variable('W' + str(i), filter_shape, tf.float32,
                                    tf.contrib.layers.variance_scaling_initializer(factor=1.0, mode='FAN_IN', uniform=True))
                b = tf.get_variable('b' + str(i), [self.num_filters], tf.float32, tf.constant_initializer(0.01))
                self.kernels.append((W, b))
                self.para.append(W)
                self.para.append(b)

        self.num_filters_total = self.num_filters * len(self.filter_sizes)

        self.m_conv = self._convolution(self.m_word_embedding)
        self.n_conv = self._convolution(self.n_word_embedding)

        # max pooling
        with tf.name_scope('max_pooling'):
            self.m_pooling = tf.reshape(self._max_pooling(self.m_conv, self.max_input_word), [-1, self.num_filters_total])
            self.n_pooling = tf.reshape(self._max_pooling(self.n_conv, self.max_input_word), [-1, self.num_filters_total])

        with tf.variable_scope("similarity"):
            W = tf.get_variable(
                name="W",
                shape=[self.num_filters_total, self.num_filters_total],
                initializer=tf.contrib.layers.xavier_initializer()
            )
            self.transform_left = tf.matmul(self.m_pooling, W)
            self.sims = tf.reduce_sum(tf.multiply(self.transform_left, self.n_pooling), axis=1, keep_dims=True)
            self.para.append(W)
            self.see = W

        self.feature = tf.concat([self.m_pooling, self.n_pooling], axis=1, name="feature")

        with tf.name_scope('neural_network'):
            W = tf.get_variable(
                "W_hidden",
                shape=[2 * self.num_filters_total, self.hidden_num],
                initializer=tf.contrib.layers.xavier_initializer()
            )
            b = tf.get_variable('b_hidden', shape=[self.hidden_num], initializer=tf.random_normal_initializer())
            self.para.append(W)
            self.para.append(b)
            self.hidden_output = tf.nn.relu(tf.nn.xw_plus_b(self.feature, W, b, name="hidden_output"))

        with tf.name_scope('dropout'):
            self.h_drop = tf.nn.dropout(self.hidden_output, self.dropout_keep_prob, name="hidden_ouput_drop")

        with tf.name_scope('output'):
            W = tf.get_variable(
                "W_output",
                shape=[self.hidden_num, 2],
                initializer=tf.contrib.layers.xavier_initializer()
            )
            b = tf.get_variable('b_output', shape=[2], initializer=tf.random_normal_initializer())
            self.para.append(W)
            self.para.append(b)
            self.logits = tf.nn.xw_plus_b(self.h_drop, W, b, name="scores")
            self.scores = tf.nn.softmax(self.logits)
            self.predictions = tf.argmax(self.scores, 1, name="predictions")

    def _add_train_op(self):
        l2_loss = tf.constant(0.0)
        for p in self.para:
            l2_loss += tf.nn.l2_loss(p)
        with tf.name_scope("loss"):
            losses = tf.nn.softmax_cross_entropy_with_logits(logits=self.logits, labels=self.y)
            self.loss_op = tf.reduce_mean(losses) + self.l2_reg_lambda * l2_loss

        # Accuracy
        with tf.name_scope("accuracy"):
            correct_predictions = tf.equal(self.predictions, tf.argmax(self.y, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")

        optimizer = tf.train.AdamOptimizer(self.learning_rate)
        self.train_op = optimizer.minimize(self.loss_op)

    def train(self):
        self._train(self.num_epochs)

    def _train(self, epochs, patience=4):

        if not os.path.exists(MODEL_PATH):
            os.makedirs(MODEL_PATH)

        saver = tf.train.Saver(max_to_keep=1)

        timer = Timer()
        timer.start('Start training')

        best_acc = 0
        nepoch_noimp = 0
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())

            num_batch_train = len(self.train_idx) // self.batch_size + 1
            for e in range(epochs):
                total_train_loss = 0
                total_train_acc = 0
                c = 0
                train_idx = shuffle(self.train_idx)
                for idx, batch in enumerate(self._next_batch(data=train_idx, num_batch=num_batch_train)):
                    feed_dict = self._make_feed_dict(batch, self.dropout_keep_prob)
                    _, train_loss, predictions, input_y, train_acc = sess.run([self.train_op, self.loss_op,
                                                                               self.predictions, self.y,
                                                                               self.accuracy],
                                                                              feed_dict=feed_dict)
                    total_train_loss += train_loss
                    total_train_acc += train_acc
                    c += 1

                    if idx % 200 == 0:
                        print("Iter {} - Loss: {} - Acc: {}".format(idx, total_train_loss / c, total_train_acc / c))
                print("End epochs {} ".format(e + 1))

                if self.is_early_stopping:
                    num_batch_val = len(self.dev_idx) // self.batch_size + 1
                    total_val_loss = 0
                    total_val_acc = 0
                    c = 0
                    for idx, batch in enumerate(self._next_batch(data=self.dev_idx, num_batch=num_batch_val)):
                        feed_dict = self._make_feed_dict(batch, self.dropout_keep_prob)
                        _, dev_loss, dev_acc = sess.run([self.train_op, self.loss_op, self.accuracy], feed_dict=feed_dict)
                        total_val_loss += dev_loss
                        total_val_acc += dev_acc
                        c += 1
                    avg_loss = total_val_loss / c
                    avg_acc = total_val_acc / c
                    print("Validation Loss: {} - Acc: {}".format(avg_loss, avg_acc))

                    if avg_acc > best_acc:
                        saver.save(sess, MODEL_PATH)
                        print('Saved the model at epoch {}'.format(e + 1))
                        best_acc = avg_acc
                        nepoch_noimp = 0
                    else:
                        nepoch_noimp += 1
                        print("Number of epochs with no improvement: {}".format(nepoch_noimp))
                        if nepoch_noimp >= patience:
                            break
            if not self.is_early_stopping:
                saver.save(sess, MODEL_PATH)
                print("Saved the model")

        timer.stop()

    def _next_batch(self, data, num_batch):
        start = 0
        c = 0

        while c < num_batch:
            index_batch = data[start:start + self.batch_size]

            X_m_batch = []
            X_n_batch = []
            Y_batch = []
            IDX_batch = []

            for i in index_batch:
                num = len(self.data[i]['y'])
                y_data = self.data[i]['y']
                for y in y_data:
                    if y == 1:
                        Y_batch.append([0, 1])
                    else:
                        Y_batch.append([1, 0])
                IDX_batch.extend([i] * num)
                X_n_batch.extend(self.data[i]['x_n'])
                X_m_batch.extend([self.data[i]['x_m']] * num)

            X_m_batch, X_n_batch, Y_batch, IDX_batch = shuffle(X_m_batch, X_n_batch, Y_batch, IDX_batch)

            m_word_ids = self._get_word_ids(X_m_batch)
            n_word_ids = self._get_word_ids(X_n_batch)

            start += self.batch_size
            c +=1
            yield(m_word_ids, n_word_ids, Y_batch, IDX_batch)

    def _make_feed_dict(self, data, dropout_keep_prob):
        return {
            self.m_word_ids: data[0],
            self.n_word_ids: data[1],
            self.y: data[2],
            self.idx: data[3],
            self.dropout_keep_op: dropout_keep_prob
        }

    def transform(self):
        def one_batch(data):
            X_m_batch = []
            X_n_batch = []
            Y_batch = []

            for i in data:
                num = len(self.data[i]['y'])
                y_data = self.data[i]['y']
                for y in y_data:
                    if y == 1:
                        Y_batch.append([0, 1])
                    else:
                        Y_batch.append([1, 0])
                X_n_batch.extend(self.data[i]['x_n'])
                X_m_batch.extend([self.data[i]['x_m']] * num)

            X_m_batch, X_n_batch, Y_batch = shuffle(X_m_batch, X_n_batch, Y_batch)

            m_word_ids = self._get_word_ids(X_m_batch)
            n_word_ids = self._get_word_ids(X_n_batch)

            yield(m_word_ids, n_word_ids, Y_batch)

        def make_feed_dict(data):
            return {
                self.m_word_ids: data[0],
                self.n_word_ids: data[1],
                self.y: data[2],
                self.dropout_keep_op: 1.0
            }
        saver = tf.train.Saver()
        with tf.Session() as sess:
            saver.restore(sess, tf.train.latest_checkpoint('development/Pointwise/data/trained_models/pointwise_100'))

            self.train_idx = shuffle(self.train_idx)

            for idx, batch in enumerate(one_batch(self.train_idx)):
                feed_dict = make_feed_dict(batch)
                predict, acc = sess.run([self.predictions, self.accuracy], feed_dict=feed_dict)

                print(acc)

    def _get_word_ids(self, X):
        word_ids = [x[1] for x in X]
        word_ids, _ = pad_sequences(word_ids, pad_tok=0, fix_length=self.max_input_word)
        return word_ids

    def _get_embedding(self, embeddings, ids):
        embed = tf.nn.embedding_lookup(embeddings, ids)
        embed = tf.reshape(embed, [-1, self.max_input_word, self.word_embedding_dim, 1])
        return embed

    def _convolution(self, embedding):
        cnn_outputs = []
        for i, filter_size in enumerate(self.filter_sizes):
            conv = tf.nn.conv2d(
                embedding,
                self.kernels[i][0],
                strides=[1, 1, self.word_embedding_dim, 1],
                padding='SAME',
                name='conv-1'
            )
            h = tf.nn.relu(tf.nn.bias_add(conv, self.kernels[i][1]), name='relu-1')
            cnn_outputs.append(h)
        cnn_reshaped = tf.concat(cnn_outputs, 3)
        return cnn_reshaped

    def _max_pooling(self, conv, input_length):
        pooled = tf.nn.max_pool(
            conv,
            ksize=[1, input_length, 1, 1],
            strides=[1, 1, 1, 1],
            padding='VALID',
            name="pool")
        return pooled


import logging

from django.conf import settings

from geodata.models import Country, Region
from iati.parser.exceptions import (
    FieldValidationError, ParserError, RequiredFieldError
)
from iati.parser.iati_parser import IatiParser
from iati_codelists import models as codelist_models
from iati_organisation.models import (
    DocumentLinkRecipientCountry, DocumentLinkTitle, Organisation,
    OrganisationDocumentLink, OrganisationDocumentLinkCategory,
    OrganisationDocumentLinkLanguage, OrganisationName, OrganisationNarrative,
    OrganisationReportingOrganisation, RecipientCountryBudget,
    RecipientCountryBudgetLine, RecipientOrgBudget, RecipientOrgBudgetLine,
    RecipientRegionBudget, RecipientRegionBudgetLine, TotalBudget,
    TotalBudgetLine, TotalExpenditure, TotalExpenditureLine
)
from iati_organisation.parser import post_save
from iati_vocabulary.models import RegionVocabulary
from solr.organisation.tasks import OrganisationTaskIndexing

# Get an instance of a logger
logger = logging.getLogger(__name__)

class Parse(IatiParser):
    """
    # NOTE: This parsed version is only covered the DataSet Organisation file
    as the following:
    - Total Budget
    -- Total Budget Line
    - Total Expenditure
    -- Total Expenditure Line
    - Organisation Document Link

    # TODO: Cover others element as The IATI Organisation File Standard
    http://reference.iatistandard.org/202/organisation-standard/overview/organisation-file/
    """

    organisation_identifier = ''

    def __init__(self, *args, **kwargs):
        super(Parse, self).__init__(*args, **kwargs)
        self.VERSION = '2.02'

        # default currency
        self.default_currency = None

        # We need a current index to put the current model
        # on the process parse
        self.organisation_document_link_current_index = 0
        self.document_link_title_current_index = 0
        self.document_link_language_current_index = 0
        self.total_budget_current_index = 0
        self.total_budget_line_current_index = 0
        self.total_expenditure_current_index = 0
        self.total_expenditure_line_current_index = 0

    def add_narrative(self, element, parent):
        default_lang = self.default_lang  # set on activity (if set)
        lang = element.attrib.get('{http://www.w3.org/XML/1998/namespace}lang')
        text = element.text

        language = self.get_or_none(codelist_models.Language, code=lang)

        if not language:
            language = default_lang

        if not parent:
            raise ParserError(
                "Unknown",
                "Narrative",
                "parent object must be passed")

        register_name = parent.__class__.__name__ + "Narrative"

        if not language:
            raise RequiredFieldError(
                register_name,
                "xml:lang",
                "must specify xml:lang on iati-activity or xml:lang on \
                        the element itself")
        if not text:
            raise RequiredFieldError(
                register_name,
                "text",
                "empty narrative")

        narrative = OrganisationNarrative()
        narrative.language = language
        narrative.content = element.text
        # This (instead of narrative.related_object) is required, otherwise
        # related object doesn't get passed to the model_store (memory) and
        # 'update_related()' fails.
        # It should probably be passed to the __init__() ?
        setattr(narrative, '_related_object', parent)

        narrative.organisation = self.get_model('Organisation')

        # TODO: handle this differently (also: breaks tests)
        self.register_model(register_name, narrative)

    def _get_currency_or_raise(self, model_name, currency):
        """
        get default currency if not available for currency-related fields
        """
        if not currency:
            currency = getattr(self.get_model(
                'Organisation'), 'default_currency')
            if not currency:
                raise RequiredFieldError(
                    model_name,
                    "currency",
                    "must specify default-currency on iati-organisation or \
                        as currency on the element itself")

        return currency

    def iati_organisations__iati_organisation(self, element):
        id = element.xpath('organisation-identifier/text()')[0]
        normalized_id = self._normalize(id)
        last_updated_datetime = self.validate_date(
            element.attrib.get('last-updated-datetime'))
        # default is here to make it default to settings 'DEFAULT_LANG' on no
        # language set (validation error we want to be flexible per instance)

        default_lang_code = element.attrib.get(
            '{http://www.w3.org/XML/1998/namespace}lang',
            settings.DEFAULT_LANG)
        if default_lang_code:
            default_lang_code = default_lang_code.lower()

        default_lang = self.get_or_none(
            codelist_models.Language,
            code=default_lang_code
        )

        default_currency = self.get_or_none(
            codelist_models.Currency,
            code=element.attrib.get('default-currency'))

        if not id:
            raise RequiredFieldError(
                "",
                "id",
                "organisation: must contain organisation-identifier")

        # TODO: check for last-updated-datetime - 2017-03-27
        old_organisation = self.get_or_none(
            Organisation, organisation_identifier=id)

        if old_organisation:
            OrganisationName.objects.filter(
                organisation=old_organisation).delete()
            OrganisationReportingOrganisation.objects.filter(
                organisation=old_organisation).delete()
            TotalBudget.objects.filter(
                organisation=old_organisation).delete()
            RecipientOrgBudget.objects.filter(
                organisation=old_organisation).delete()
            RecipientCountryBudget.objects.filter(
                organisation=old_organisation).delete()
            RecipientRegionBudget.objects.filter(
                organisation=old_organisation).delete()
            TotalExpenditure.objects.filter(
                organisation=old_organisation).delete()
            OrganisationDocumentLink.objects.filter(
                organisation=old_organisation).delete()

            organisation = old_organisation
        else:
            organisation = Organisation()

        organisation.organisation_identifier = id
        organisation.normalized_organisation_identifier = normalized_id
        organisation.last_updated_datetime = last_updated_datetime
        organisation.default_lang = default_lang
        organisation.iati_standard_version_id = self.VERSION
        organisation.default_currency = default_currency

        organisation.published = True
        organisation.ready_to_publish = True
        organisation.modified = False
        organisation.dataset = self.dataset

        self.organisation_identifier = organisation.organisation_identifier
        self.default_currency = default_currency

        # for later reference
        self.default_lang = default_lang

        self.register_model('Organisation', organisation)

        return element

    def iati_organisations__iati_organisation__organisation_identifier(
            self, element):
        # already set in iati_organisation
        return element

    def iati_organisations__iati_organisation__name(self, element):
        name_list = self.get_model_list('OrganisationName')

        if name_list and len(name_list) > 0:
            raise FieldValidationError(
                "name", "Duplicate names are not allowed")

        organisation = self.get_model('Organisation')

        name = OrganisationName()
        name.organisation = organisation

        self.register_model('OrganisationName', name)

        return element

    def iati_organisations__iati_organisation__name__narrative(self, element):
        model = self.get_model('OrganisationName')
        self.add_narrative(element, model)

        if element.text:

            organisation = self.get_model('Organisation')

            if organisation.primary_name:
                default_lang = self.default_lang  # set on activity (if set)
                lang = element.attrib.get(
                    '{http://www.w3.org/XML/1998/namespace}lang', default_lang)
                if lang == 'en':
                    organisation.primary_name = element.text
            else:
                organisation.primary_name = element.text
        return element

    def iati_organisations__iati_organisation__reporting_org(self, element):
        # Although OrganisationReportingOrganisation and Organisation has
        # One-to-One relation on the database level, we check here whether
        # element 'reporting-org' occurs only once in the parent element
        # 'organisation'.
        organisation = self.get_model('Organisation')
        if 'OrganisationReportingOrganisation' in self.model_store:
            for reporting_org in self.model_store[
                    'OrganisationReportingOrganisation']:
                if reporting_org.organisation == organisation:
                    raise ParserError("Organisation",
                                      "OrganisationReportingOrganisation",
                                      "must occur no more than once.")
        # narrative is parsed in different method but as it is required
        # sub-element in 'name' element so we check it here.
        narrative = element.xpath("narrative")
        if len(narrative) < 1:
            raise RequiredFieldError("OrganisationName", "narrative",
                                     "must occur at least once.")
        reporting_org_identifier = element.attrib.get("ref")
        if reporting_org_identifier is None:
            raise RequiredFieldError("OrganisationReportingOrganisation",
                                     "ref", "required field missing.")
        org_type = element.attrib.get("type")
        if org_type is None:
            raise RequiredFieldError("OrganisationReportingOrganisation",
                                     "type", "required field missing.")
        # here org_type is OrganisationType object.
        org_type = self.get_or_none(codelist_models.OrganisationType,
                                    code=org_type)
        if org_type is None:
            raise FieldValidationError(
                "OrganisationReportingOrganisation",
                "type",
                "not found on the accompanying codelist.",
                None,
                None,
            )
        secondary_reporter = self.makeBool(element.attrib.get(
            "secondary-reporter"))

        reporting_org = OrganisationReportingOrganisation()
        reporting_org.organisation = organisation
        reporting_org.org_type = org_type
        reporting_org.secondary_reporter = secondary_reporter
        reporting_org.reporting_org_identifier = reporting_org_identifier

        self.register_model("OrganisationReportingOrganisation", reporting_org)
        return element

    def iati_organisations__iati_organisation__reporting_org__narrative(
            self, element):
        """atributes:

        tag:narrative"""
        model = self.get_model('OrganisationReportingOrganisation')
        self.add_narrative(element, model)
        return element

    def iati_organisations__iati_organisation__total_budget(self, element):
        """atributes:

        tag:total-budget"""
        status = self.get_or_none(
            codelist_models.BudgetStatus, code=element.attrib.get('status'))

        model = self.get_model('Organisation')
        total_budget = TotalBudget()
        total_budget.organisation = model

        if status:
            total_budget.status = status

        self.total_budget_current_index = \
            self.register_model('TotalBudget', total_budget)

        # store element
        return element

    def iati_organisations__iati_organisation__total_budget__period_start(
            self, element):
        """atributes:
        iso-date:2014-01-01

        tag:period-start"""
        model = self.get_model('TotalBudget', self.total_budget_current_index)
        model.period_start = self.validate_date(element.attrib.get('iso-date'))

        # store element
        return element

    def iati_organisations__iati_organisation__total_budget__period_end(
            self, element):
        """atributes:
        iso-date:2014-12-31

        tag:period-end"""
        model = self.get_model('TotalBudget', self.total_budget_current_index)
        model.period_end = self.validate_date(element.attrib.get('iso-date'))
        # store element
        return element

    def iati_organisations__iati_organisation__total_budget__value(
            self, element):
        """atributes:
        currency:USD
        value-date:2014-01-0

        tag:value"""
        model = self.get_model('TotalBudget', self.total_budget_current_index)
        model.currency = self.get_or_none(
            codelist_models.Currency,
            code=self._get_currency_or_raise(
                'total-budget/value',
                element.attrib.get('currency')))
        model.value_date = self.validate_date(element.attrib.get('value-date'))
        model.value = element.text
        # store element
        return element

    def iati_organisations__iati_organisation__total_budget__budget_line(
            self, element):
        """atributes:
        ref:1234

        tag:budget-line"""
        budget_line = TotalBudgetLine()
        budget_line.ref = element.attrib.get('ref', '-')

        budget_line.total_budget = self.get_model(
            'TotalBudget', self.total_budget_current_index)

        self.total_budget_line_current_index = \
            self.register_model('TotalBudgetLine', budget_line)
        # store element
        return element

    def iati_organisations__iati_organisation__total_budget__budget_line__value(self, element):  # NOQA: E501
        """atributes:
        currency:USD
        value-date:2014-01-01

        tag:value"""
        model = self.get_model('TotalBudgetLine',
                               self.total_budget_line_current_index)

        model.currency = self.get_or_none(
            codelist_models.Currency,
            code=self._get_currency_or_raise(
                'total-budget/budget-line/value',
                element.attrib.get('currency')))
        model.value = element.text
        model.value_date = self.validate_date(element.attrib.get('value-date'))
        # store element
        return element

    def iati_organisations__iati_organisation__total_budget__budget_line__narrative(self, element):  # NOQA: E501
        """atributes:

        tag:narrative"""
        model = self.get_model('TotalBudgetLine',
                               self.total_budget_line_current_index)

        self.add_narrative(element, model)
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_org_budget(
            self, element):
        """atributes:

        tag:recipient-org-budget"""
        status = self.get_or_none(
            codelist_models.BudgetStatus, code=element.attrib.get('status'))

        model = self.get_model('Organisation')
        recipient_org_budget = RecipientOrgBudget()
        recipient_org_budget.organisation = model
        if status:
            recipient_org_budget.status = status
        self.register_model('RecipientOrgBudget', recipient_org_budget)
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_org_budget__recipient_org(self, element):  # NOQA: E501
        """atributes:
        ref:AA-ABC-1234567

        tag:recipient-org"""
        model = self.get_model('RecipientOrgBudget')
        model.recipient_org_identifier = element.attrib.get('ref')
        if Organisation.objects.filter(
            organisation_identifier=element.attrib.get('ref')
        ).exists():
            model.recipient_org = Organisation.objects.get(
                pk=element.attrib.get('ref'))

        # store element
        return element

    def iati_organisations__iati_organisation__recipient_org_budget__recipient_org__narrative(  # NOQA: E501
            self, element):
        """atributes:

        tag:narrative"""
        model = self.get_model('RecipientOrgBudget')
        self.add_narrative(element, model)
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_org_budget__period_start(self, element):  # NOQA: E501
        """atributes:
        iso-date:2014-01-01

        tag:period-start"""
        model = self.get_model('RecipientOrgBudget')
        model.period_start = self.validate_date(element.attrib.get('iso-date'))

        # store element
        return element

    def iati_organisations__iati_organisation__recipient_org_budget__period_end(self, element):  # NOQA: E501
        """atributes:
        iso-date:2014-12-31

        tag:period-end"""
        model = self.get_model('RecipientOrgBudget')
        model.period_end = self.validate_date(element.attrib.get('iso-date'))
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_org_budget__value(
            self, element):
        """atributes:
        currency:USD
        value-date:2014-01-01

        tag:value"""
        model = self.get_model('RecipientOrgBudget')
        model.currency = self.get_or_none(
            codelist_models.Currency,
            code=self._get_currency_or_raise(
                'recipient-org-budget/value',
                element.attrib.get('currency')))
        value_date = element.attrib.get('value-date')
        if value_date is None:
            raise RequiredFieldError("RecipientOrgBudget", "value-date",
                                     "required field missing.")
        value_date = self.validate_date(value_date)
        if not value_date:
            raise FieldValidationError(
                "RecipientOrgBudget",
                "value-date",
                "not in the correct range.",
                None,
                None,
            )
        model.value_date = value_date
        model.value = element.text
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_org_budget__budget_line(self, element):  # NOQA: E501
        """atributes:
        ref:1234

        tag:budget-line"""
        org_budget = self.get_model('RecipientOrgBudget')
        budget_line = RecipientOrgBudgetLine()
        budget_line.ref = element.attrib.get('ref')
        budget_line.recipient_org_budget = org_budget
        self.register_model('RecipientOrgBudgetLine', budget_line)
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_org_budget__budget_line__value(  # NOQA: E501
            self, element):
        """atributes:
        currency:USD
        value-date:2014-01-01

        tag:value"""
        model = self.get_model('RecipientOrgBudgetLine')
        model.currency = self.get_or_none(
            codelist_models.Currency,
            code=self._get_currency_or_raise(
                'recipient-org-budget/budget-line/value',
                element.attrib.get('currency')))
        model.value = element.text
        model.value_date = self.validate_date(element.attrib.get('value-date'))
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_org_budget__budget_line__narrative(  # NOQA: E501
            self, element):
        """atributes:

        tag:narrative"""
        model = self.get_model('RecipientOrgBudgetLine')
        self.add_narrative(element, model)
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_country_budget(
            self, element):
        """atributes:

        tag:recipient-country-budget"""
        status = self.get_or_none(
            codelist_models.BudgetStatus, code=element.attrib.get('status'))
        model = self.get_model('Organisation')
        recipient_country_budget = RecipientCountryBudget()
        recipient_country_budget.organisation = model
        if status:
            recipient_country_budget.status = status
        self.register_model('RecipientCountryBudget', recipient_country_budget)
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_country_budget__recipient_country(  # NOQA: E501
            self, element):
        """atributes:
        code:AF

        tag:recipient-country"""
        model = self.get_model('RecipientCountryBudget')
        model.country = self.get_or_none(
            Country, code=element.attrib.get('code'))

        # store element
        return element

    def iati_organisations__iati_organisation__recipient_country_budget__recipient_country__narrative(self, element):  # NOQA: E501
        recipient_country_budget = self.get_model(
            'RecipientCountryBudget'
        )
        self.add_narrative(element, recipient_country_budget)

        return element

    def iati_organisations__iati_organisation__recipient_country_budget__period_start(  # NOQA: E501
            self, element):
        """atributes:
        iso-date:2014-01-01

        tag:period-start"""
        model = self.get_model('RecipientCountryBudget')
        model.period_start = self.validate_date(element.attrib.get('iso-date'))
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_country_budget__period_end(self, element):  # NOQA: E501
        """atributes:
        iso-date:2014-12-31

        tag:period-end"""
        model = self.get_model('RecipientCountryBudget')
        model.period_end = self.validate_date(element.attrib.get('iso-date'))
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_country_budget__value(
            self, element):
        """atributes:
        currency:USD
        value-date:2014-01-01

        tag:value"""
        model = self.get_model('RecipientCountryBudget')
        model.currency = self.get_or_none(
            codelist_models.Currency,
            code=self._get_currency_or_raise(
                'recipient-country-budget/value',
                element.attrib.get('currency')))

        value_date = element.attrib.get("value-date")
        if value_date is None:
            raise RequiredFieldError(
                "RecipientCountryBudget",
                "value-date",
                "required field missing."
            )

        value_date = self.validate_date(value_date)
        if not value_date:
            raise FieldValidationError(
                "RecipientCountryBudget",
                "value-date",
                "not in the correct range.",
                None,
                None,
            )

        model.value = element.text
        model.value_date = value_date
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_country_budget__budget_line(  # NOQA: E501
            self, element):
        """atributes:
        ref:1234

        tag:budget-line"""
        recipient_country_budget = self.get_model('RecipientCountryBudget')
        budget_line = RecipientCountryBudgetLine()
        budget_line.ref = element.attrib.get('ref', '-')
        budget_line.recipient_country_budget = recipient_country_budget
        self.register_model('RecipientCountryBudgetLine', budget_line)
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_country_budget__budget_line__value(  # NOQA: E501
            self, element):
        """atributes:
        currency:USD
        value-date:2014-01-01

        tag:value"""
        model = self.get_model('RecipientCountryBudgetLine')
        model.currency = self.get_or_none(
            codelist_models.Currency,
            code=self._get_currency_or_raise(
                'recipient-country-budget/budget-line/value',
                element.attrib.get('currency')))
        model.value = element.text
        model.value_date = self.validate_date(element.attrib.get('value-date'))
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_country_budget__budget_line__narrative(  # NOQA: E501
            self, element):
        """atributes:

        tag:narrative"""
        model = self.get_model('RecipientCountryBudgetLine')
        self.add_narrative(element, model)
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_region_budget(
            self, element):
        """atributes:

        tag:recipient-region-budget"""
        status = self.get_or_none(
            codelist_models.BudgetStatus, code=element.attrib.get('status'))
        model = self.get_model('Organisation')
        recipient_region_budget = RecipientRegionBudget()
        recipient_region_budget.organisation = model
        if status:
            recipient_region_budget.status = status
        self.register_model('RecipientRegionBudget', recipient_region_budget)
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_region_budget__recipient_region(  # NOQA: E501
            self, element):
        """atributes:
        code:AF

        tag:recipient-region"""
        model = self.get_model('RecipientRegionBudget')

        # TODO: make defaults more transparant, here: 'OECD-DAC default'
        vocabulary = self.get_or_none(
            RegionVocabulary, code=element.attrib.get('vocabulary', '1'))
        vocabulary_uri = element.attrib.get('vocabulary-uri')

        model.region = Region.objects.filter(code=element.attrib.get(
            'code'), region_vocabulary=vocabulary).first()

        if not vocabulary:
            raise RequiredFieldError(
                "recipient-region-budget",
                "recipient-region",
                "invalid vocabulary")

        model.vocabulary = vocabulary
        model.vocabulary_uri = vocabulary_uri

        # store element
        return element

    def iati_organistions__iati_organisation__recipient_region_budget__recipient_region__narrative(self, element):  # NOQA: E501
        recipient_region_budget = self.get_model('RecipientRegionBudget')
        self.add_narrative(element, recipient_region_budget)
        return element

    def iati_organisations__iati_organisation__recipient_region_budget__period_start(  # NOQA: E501
            self, element):
        """atributes:
        iso-date:2014-01-01

        tag:period-start"""
        model = self.get_model('RecipientRegionBudget')
        model.period_start = self.validate_date(element.attrib.get('iso-date'))
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_region_budget__period_end(self, element):  # NOQA: E501
        """atributes:
        iso-date:2014-12-31

        tag:period-end"""
        model = self.get_model('RecipientRegionBudget')
        model.period_end = self.validate_date(element.attrib.get('iso-date'))
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_region_budget__value(
            self, element):
        """atributes:
        currency:USD
        value-date:2014-01-01

        tag:value"""
        model = self.get_model('RecipientRegionBudget')
        model.currency = self.get_or_none(
            codelist_models.Currency,
            code=self._get_currency_or_raise(
                'recipient-region-budget/value',
                element.attrib.get('currency')))
        value_date = element.attrib.get('value-date')
        if value_date is None:
            raise RequiredFieldError(
                "RecipientRegionBudget",
                "value-date",
                "required field missing."
            )
        value_date = self.validate_date(value_date)
        if not value_date:
            raise FieldValidationError(
                "RecipientRegionBudget",
                "value-date",
                "not in the correct range.",
                None,
                None,
            )
        model.value = element.text
        model.value_date = value_date
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_region_budget__budget_line(self, element):  # NOQA: E501
        """atributes:
        ref:1234

        tag:budget-line"""
        recipient_region_budget = self.get_model('RecipientRegionBudget')
        budget_line = RecipientRegionBudgetLine()
        budget_line.ref = element.attrib.get('ref')
        budget_line.recipient_region_budget = recipient_region_budget
        self.register_model('RecipientRegionBudgetLine', budget_line)
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_region_budget__budget_line__value(  # NOQA: E501
            self, element):
        """atributes:
        currency:USD
        value-date:2014-01-01

        tag:value"""
        model = self.get_model('RecipientRegionBudgetLine')
        model.currency = self.get_or_none(
            codelist_models.Currency,
            code=self._get_currency_or_raise(
                'recipient-region-budget/budget-line/value',
                element.attrib.get('currency')))
        model.value = element.text
        model.value_date = self.validate_date(element.attrib.get('value-date'))
        # store element
        return element

    def iati_organisations__iati_organisation__recipient_region_budget__budget_line__narrative(  # NOQA: E501
            self, element):
        """atributes:

        tag:narrative"""
        model = self.get_model('RecipientRegionBudgetLine')
        self.add_narrative(element, model)
        # store element
        return element

    def iati_organisations__iati_organisation__total_expenditure(
            self, element):
        """
        """
        model = self.get_model('Organisation')
        total_expenditure = TotalExpenditure()
        total_expenditure.organisation = model

        self.total_expenditure_current_index = \
            self.register_model('TotalExpenditure', total_expenditure)

        return element

    def iati_organisations__iati_organisation__total_expenditure__period_start(
            self, element):
        """
        """
        model = self.get_model(
            'TotalExpenditure', self.total_expenditure_current_index)
        model.period_start = self.validate_date(element.attrib.get('iso-date'))
        return element

    def iati_organisations__iati_organisation__total_expenditure__period_end(
            self, element):
        """
        """
        model = self.get_model(
            'TotalExpenditure', self.total_expenditure_current_index)
        model.period_end = self.validate_date(element.attrib.get('iso-date'))
        return element

    def iati_organisations__iati_organisation__total_expenditure__value(
            self, element):
        """
        """
        model = self.get_model(
            'TotalExpenditure', self.total_expenditure_current_index)
        code = element.attrib.get('currency')
        currency = self.get_or_none(codelist_models.Currency, code=code)

        if not code:
            raise RequiredFieldError(
                "total-expenditure/value/currency",
                "code",
                "required element empty")

        if not currency:
            raise FieldValidationError(
                "total-expenditure/value/currency",
                "code",
                "not found on the accompanying code list")

        model.value_date = self.validate_date(element.attrib.get('value-date'))
        model.currency = currency
        model.value = element.text
        return element

    def iati_organisations__iati_organisation__total_expenditure__expense_line(
            self, element):
        """
        """
        budget_line = TotalExpenditureLine()
        budget_line.ref = element.attrib.get('ref', '')
        budget_line.total_expenditure = \
            self.get_model(
                'TotalExpenditure', self.total_expenditure_current_index)

        self.total_expenditure_line_current_index = \
            self.register_model('TotalExpenditureBudgetLine', budget_line)

        return element

    def iati_organisations__iati_organisation__total_expenditure__expense_line__value(  # NOQA: E501
            self, element):
        """
        """
        model = self.get_model('TotalExpenditureBudgetLine',
                               self.total_expenditure_line_current_index)

        currency = element.attrib.get("currency")
        if not currency:
            currency = getattr(
                self.get_model("Organisation"),
                "default_currency"
            )
            if not currency:
                raise RequiredFieldError(
                    "TotalExpenditureLine",
                    "currency",
                    "must specify default-currency on iati-activity or as "
                    "currency on the element itself."
                )

        else:
            currency = self.get_or_none(
                codelist_models.Currency,
                code=currency
            )
            if currency is None:
                raise FieldValidationError(
                    "TotalExpenditureLine",
                    "currency",
                    "not found on the accompanying codelist.",
                    None,
                    None,
                )
        model.value_date = self.validate_date(element.attrib.get('value-date'))
        model.currency = currency
        model.value = element.text
        return element

    def iati_organisations__iati_organisation__total_expenditure__expense_line__narrative(  # NOQA: E501
            self, element):
        """
        """
        model = self.get_model('TotalExpenditureBudgetLine',
                               self.total_expenditure_line_current_index)

        self.add_narrative(element, model)
        return element

    def iati_organisations__iati_organisation__document_link(self, element):
        """atributes:
        format:application/vnd.oasis.opendocument.text
        url:http:www.example.org/docs/report_en.odt

        tag:document-link"""
        model = self.get_model('Organisation')
        document_link = OrganisationDocumentLink()
        document_link.organisation = model
        document_link.url = element.attrib.get('url')
        document_link.file_format = self.get_or_none(
            codelist_models.FileFormat, code=element.attrib.get('format'))

        # Set the document link on the process
        self.organisation_document_link_current_index = \
            self.register_model('OrganisationDocumentLink', document_link)

        # store element
        return element

    def iati_organisations__iati_organisation__document_link__title(
            self, element):
        """atributes:

        tag:title"""

        document_link_title = DocumentLinkTitle()
        self.document_link_title_current_index = \
            self.register_model('DocumentLinkTitle', document_link_title)

        model = self.get_model('OrganisationDocumentLink',
                               self.organisation_document_link_current_index)
        document_link_title.document_link = model

        # store element
        return element

    def iati_organisations__iati_organisation__document_link__title__narrative(
            self, element):
        """atributes:

    tag:narrative"""
        model = self.get_model('DocumentLinkTitle',
                               self.document_link_title_current_index)

        self.add_narrative(element, model)
        # store element
        return element

    def iati_organisations__iati_organisation__document_link__category(
            self, element):
        """atributes:
        code:B01

        tag:category"""
        model = self.get_model('OrganisationDocumentLink',
                               self.organisation_document_link_current_index)

        document_category = self.get_or_none(
            codelist_models.DocumentCategory,
            code=element.attrib.get('code'))

        document_link_category = OrganisationDocumentLinkCategory()
        document_link_category.category = document_category
        document_link_category.document_link = model

        self.register_model(
            'OrganisationDocumentLinkCategory', document_link_category)

        return element

    def iati_organisations__iati_organisation__document_link__language(
            self, element):
        """atributes:
        code:en

        tag:language"""
        organisation_document_link_language = \
            OrganisationDocumentLinkLanguage()

        organisation_document_link_language.language = self.get_or_none(
            codelist_models.Language,
            code=element.attrib.get('code'))

        model = self.get_model('OrganisationDocumentLink',
                               self.organisation_document_link_current_index)
        organisation_document_link_language.document_link = model

        self.document_link_language_current_index = self.register_model(
            'OrganisationDocumentLinkLanguage',
            organisation_document_link_language)

        # store element
        return element

    def iati_organisations__iati_organisation__document_link__document_date(
            self, element):
        """attributes:
        format:application/vnd.oasis.opendocument.text
        url:http:www.example.org/docs/report_en.odt

        tag:document-link"""
        iso_date = element.attrib.get('iso-date')

        if not iso_date:
            raise RequiredFieldError(
                "document-link/document-date",
                "iso-date",
                "required attribute missing")

        iso_date = self.validate_date(iso_date)

        if not iso_date:
            raise FieldValidationError(
                "document-link/document-date",
                "iso-date",
                "iso-date not of type xsd:date")

        document_link = self.get_model(
            'OrganisationDocumentLink',
            self.organisation_document_link_current_index)

        document_link.iso_date = iso_date
        return element

    def iati_organisations__iati_organisation__document_link__recipient_country(  # NOQA: E501
            self, element):
        """atributes:
        code:AF

        tag:recipient-country"""
        model = self.get_model('OrganisationDocumentLink',
                               self.organisation_document_link_current_index)

        country = self.get_or_none(Country, code=element.attrib.get('code'))

        document_link_recipient_country = DocumentLinkRecipientCountry()
        document_link_recipient_country.recipient_country = country
        document_link_recipient_country.document_link = model

        self.register_model('DocumentLinkRecipientCountry',
                            document_link_recipient_country)

        # store element
        return element

    def post_save_models(self):
        """Perform all actions that need to happen after a single
        organisation's been parsed."""
        organisation = self.get_model('Organisation')

        if not organisation:
            return False

        post_save.set_activity_reporting_organisation(organisation)
        post_save.set_publisher_fk(organisation)

        # Solr indexing
        OrganisationTaskIndexing(instance=organisation).run()

    def post_save_file(self, xml_source):
        pass

    def post_save_validators(self, dataset):
        pass


import cv2 as cv
import numpy as np
from math import atan

# deskews picture with i pixels
# returns deskewed picture
def deskew(img, i):

    rows = img.shape[0]
    cols = img.shape[1]
    pts1 = np.float32([[0, 0], [cols - 1,  0], [cols - 1 + i, rows - 1]])
    pts2 = np.float32([[0, 0], [cols - 1, 0], [cols - 1, rows - 1]])
    M = cv.getAffineTransform(pts1,pts2)
    dst = cv.warpAffine(img,M,(cols,rows))

    return dst

# crops whitespace from image
def cropWhiteSpace(image):
    coords = cv.findNonZero(image)
    x, y, w, h = cv.boundingRect(coords)
    newImg = image[y:y+h, x:x+w]

    return newImg

def cropWhiteSpace3D(image):
    coords = cv.findNonZero(image[:,:,0])
    coords2 = (cv.findNonZero(image[:,:,1]))
    coords3 = (cv.findNonZero(image[:,:,2]))
    x, y, w, h = cv.boundingRect(coords + coords2 + coords3)
    newImg = image[y:y+h, x:x+w, :]

    return newImg
# deskews over different angles and calculates best based on
# summed squared vertical density
# returns slant angle + deslanted image
def slantNormalize(img):

    imgs = []
    ss = []
    for i in range(50):
        # pad whitespace to left border
        dst = cv.copyMakeBorder(img, 0, 0, 50, 0, cv.BORDER_CONSTANT)
        newImg = deskew(dst, i)

        # crop image
        newImg = cropWhiteSpace(newImg)
        y, x = newImg.shape

        s = 0
        for i in range(x):
            s += sum(newImg[:,i]) ** 2

        imgs.append(newImg)
        ss.append(s)

    finalS = np.argmax(ss)
    finalImg = imgs[finalS]

    y, x = finalImg.shape
    angle = atan(finalS / y)

    return angle, finalImg


import sys
from random import randint
import pygame
from utils.models import Game, GameElement

pygame.init()

game = Game(width=900, height=700, color=(32, 52, 81))
game.display.set_caption("Awesome Shooter Game")
dimentions = (game.screen.get_width(), game.screen.get_height())

fighter = GameElement('Fighter', './images/fighter.png', dimentions)
fighter.move_by_x(game.screen.get_width() / 2 - fighter.width / 2)
fighter.move_by_y(game.screen.get_height() - fighter.height)
fighter.set_step(0.5)

ball = GameElement('Ball', './images/ball.png', dimentions)
ball.set_step(0.3)
ball.hide()

alien = GameElement('Alien', './images/alien.png', dimentions)
alien.set_step(0.1)
alien.set_position(randint(0, game.screen.get_width() - alien.width), 0)

game_is_running = True

while game_is_running:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT and fighter.can_move_left():
                fighter.move_left()
            if event.key == pygame.K_RIGHT and fighter.can_move_right():
                fighter.move_right()
            if event.key == pygame.K_SPACE:
                ball.show()
                ball.move_by_x(fighter.x + fighter.width / 2 - ball.width / 2)
                ball.move_by_y(fighter.y - ball.height)
        if event.type == pygame.KEYUP:
            if event.key == pygame.K_LEFT:
                fighter.stop_moving_left()
            if event.key == pygame.K_RIGHT:
                fighter.stop_moving_right()

    fighter.keep_moving_left()
    fighter.keep_moving_right()

    alien.move_down()

    if ball.is_visible and ball.is_above_up():
        ball.hide()

    if ball.is_visible:
        ball.move_up()

    game.fill()
    game.blit_element(fighter)
    game.blit_element(alien)

    if ball.is_visible:
        game.blit_element(ball)

    game.display.update()

    if alien.is_under_down():
        game_is_running = False

game_over_text = game.font.render("Game Over", True, "white")
game_over_rectangle = game_over_text.get_rect()
game_over_rectangle.center = (game.screen.get_width() / 2, game.screen.get_height() / 2)
game.screen.blit(game_over_text, game_over_rectangle)
pygame.display.update()
pygame.time.wait(5000)

pygame.quit()


from DataManager import dataBase
from telethon import TelegramClient
import asyncio
import requests
from LogData import logger, getDateTime
from utilities import *

logger.info('default', 'Basic Startup complete!!')
userData = dataBase.getUserData()
#function to send the telegram messages
async def sendMessage(data,message):
    data['session'] = message
    print(message)
    for user in data['user']:
        try:
            await client.send_message(user, message)
            logger.info('SlotSuccess', f"to {user} {msg}")
        except Exception as e:
            logger.error('SendFail', f"{msg}\n\nUSER: {user} \nEXCEPTION: {' , '.join(e.args)}")

async def filterData(session, age, data):
    # process the data and get relevant data
    message = processData(session, age)
    #check if the message is not empty. else do not send any message on group
    if(len(message)):
        #check if the message is same as that sent before to avoid repetition of multiple same messages
        if data['session'] != message:
            await sendMessage(data, message)
        else:
            logger.warning('SlotSame', msg)
    else:
        logger.info('SlotFail', msg)

#Telegram Authorization and login function
async def login():
    account = dataBase.getTelegramCredentials()
    global client
    client = TelegramClient( account['Name'], account['Id'], account['Hash'] )
    logger.info('CliStrt')
    try:
        await client.start()
        logger.info('StartSuccess')
        return True
    except Exception as e:
        logger.error('StartFailed')
        return False

async def main():

    #try to login until successful
    while(not await login()):
        pass

    while(1):
        counter = 0
        userData = dataBase.getUserData()
        sleepTime = dataBase.getSleepTime()
        t,date =getDateTime()
        for district,data in userData.items():
            url = f"{baseURL}{district}&date={date}"
            response = requests.get(url, headers=headers)
            global msg
            msg = f"at PINCODE: {district}"

            #check if the request was processed successfully.
            if response.ok :
                session= response.json()
                for age,value in data.items():
                    if len(value['user']):
                        await filterData(session,int(age),value)

            else:
                logger.error('ApiFail', f"{msg} status code: {str(response.status_code)} REASON: {response.reason}")
            counter  += 1
            await asyncio.sleep(sleepTime)
            if(counter > 10):
                dataBase.loadData()

asyncio.run(main())


"""
Sample data created for testing purposes
"""

def raw_data() -> dict:
    """
    Raw data to mock

    :return: Dictionary with the raw data
    """
    data = {
        'unit,sex,age,geo\time': ['YR,F,Y1,AL','YR,F,Y1,AL','YR,F,Y1,PT'],
        '2021': ['79.4','79.6','79.6'],
        '2022': ['79.4','79.6','79.6'],
        '2023': ['83.2 ','79.6','79.6']
    }
    return data

def expected_result() -> dict:
    """
    Expected result to mock

    :return: Dictionary with the expected result
    """
    result = {
        'unit': ['YR','YR','YR'],
        'sex': ['F','F','F'],
        'age': ['Y1','Y1','Y1'],
        'region': ['PT','PT','PT'],
        'year': [2021,2022,2023],
        'value': [79.6,79.6,79.6]
    }
    return result


class node:
    def __init__(self, data=None):
        self.data = data
        self.next = None

class linkedList:
    def __init__(self, data=None):
        self.head = node(data)

    def append(self, data):
        newNode = node(data)
        if self.head is None:
            self.head = newNode
        else:
            last = self.head
            while (last.next):
                last = last.next
            last.next = newNode

    def display(self):
        curNode = self.head
        while (curNode):
            print(curNode.data)
            curNode = curNode.next

list1 = linkedList(0)

list1.append(10)
list1.append(20)

list1.display()


#  Finger exercise: Let s be a string that contains a sequence of decimal numbers
# separated by commas, e.g., s = '1.23,2.4,3.123'. Write a program that prints the
# sum of the numbers in s.

s = '1.23,2.4,3.123'

segmented = s.split(',')  # split the string into list elements

convertedlist = []

# convert from string to float
for num in segmented:
    num = float(num)
    convertedlist.append(num)

# total the numbers
total = 0
for num in convertedlist:
    total += num

print(total)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Interpolation of Amplitude and Phase using the interpolant derived by Zhigang Xu.
For details see - https://doi.org/10.1007/s10236-017-1122-8

TODO:
    * Add a wrapper class
    * Add type and error check
    * Add case for missing value

Author: khan
"""
import numpy as np
import matplotlib.pyplot as plt

class Point(object):
    '''
    Point(x, y=0, a=None, p=None, isradians=False) is point object to hold point information
    including the amplitude and phase.

    args:
        x (float)       :   x position
        y (float)       :   y position
        a (float)       :   amplitude
        p (float)       :   phase in degree or radians (default: degrees)
        isradians (bool):   if input is in degrees or radians (default: false)

    returns:
        An instance of Point class.

    attributes:
        x (float)       : x position
        y (float)       : y position
        a (float)       : amplitude
        p (float)       : phase in radians

    methods:
        print() : prints the attributes

    TODO:
        * add typecheck and error handling
    '''
    def __init__(self, x, y=0, a=None, p=None, isradians=False):
        self.x = float(x)
        self.y = float(y)
        if a is not None:
            self.a = float(a)
        else:
            self.a = float(0)

        self.isradians = isradians

        if p is not None:
            if self.isradians:
                self.p = float(p)
            else:
                self.p = float(p)*np.pi/180.0
        else:
            self.p = float(0)

    def print(self):
        print(self.x, self.y, self.a, self.p)

    def __lt__(self, other):
        return(self.x < other.x and self.y < other.y)

class Grid(object):
    '''
    Grid(x, y, A=None, P=None, isradians=False) is the grid object to hold points in a meshgrid.

    args:
        x ([float])         : x positon array in the structured grid
        y ([float])         : y position array in the structured grid

        A ([[float]])       : 2D array of size (x, y) containing amplitude
        P ([[float]])       : 2D array of size (x, y) containing phase
        isradians (bool)    : if the phase in in radians

    '''
    def __init__(self, x, y, A=None, P=None, isradians=False):
        # Initiating variables
        self.x = x
        self.y = y
        __X, __Y = np.meshgrid(self.x, self.y, indexing='xy')
        self.shape = __X.shape
        self.length = len(__X.flat)
        self.isradians = isradians

        if A is None:
            __A = np.zeros(shape=self.shape)
        else:
            __A = A

        if P is None:
            __P = np.zeros(shape=self.shape)
        else:
            __P = P

        # Creating point meshgrid
        self.points = np.array([Point(x=__X.flat[i], y=__Y.flat[i], a=__A.flat[i], p=__P.flat[i], isradians=self.isradians) for i in np.arange(self.length)])
        self.points = np.reshape(self.points, self.shape)

    def getpoints(self, reshaped=True):
        __points = np.array([point for point in self.points.flat])
        if reshaped:
            return(np.reshape(self.points, self.shape))
        else:
            return(self.points)

    def getx(self, reshaped=True):
        __X = np.array([point.x for point in self.points.flat])
        if reshaped:
            __X = np.reshape(__X, self.shape)
        return(__X)

    def gety(self, reshaped=True):
        __Y = np.array([point.y for point in self.points.flat])
        if reshaped:
            __Y = np.reshape(__Y, self.shape)
        return(__Y)

    def getamplitude(self, reshaped=True):
        __A = np.array([point.a for point in self.points.flat])
        if reshaped:
            __A = np.reshape(__A, self.shape)
        return(__A)

    def getphase(self, reshaped=True, degrees=False):
        __P = np.array([point.p for point in self.points.flat])
        if degrees:
            __P = __P*180/np.pi

        if reshaped:
            __P = np.reshape(__P, self.shape)
        return(__P)

    def print(self, degrees=False):
        print('X =\n', self.getx())
        print('Y =\n', self.gety())
        print('A =\n', self.getamplitude())
        print('P =\n', self.getphase(degrees=degrees))

    def plot(self, degrees=False):
        __X = self.getx(reshaped=False)
        __Y = self.gety(reshaped=False)
        __A = self.getamplitude()
        if degrees:
            __P = self.getphase(degrees=True)
            __P[__P < 0] = 360 + __P[__P < 0]
        else:
            __P = self.getphase()

        __xy = np.array([(__X[i], __Y[i]) for i in np.arange(self.length)])

        __plot = plt.subplot(121)
        __plot.matshow(__A)
        plt.title('Amplitude')
        __s = [str(i) for i in __A.flat]
        for i in np.arange(len(__s)):
            __plot.annotate(s=__s[i], xy=__xy[i], ha='center', va='center')

        __plot = plt.subplot(122)
        __plot.matshow(__P)
        plt.title('Phase')
        __s = [str(i) for i in __P.flat]
        for i in np.arange(len(__s)):
            __plot.annotate(s=__s[i], xy=__xy[i], ha='center', va='center')

        plt.show()

class Interpolator1D(object):
    '''
        Interpolator1D(points, axis=1, sort=True) creates the 1-D interpolation
        object using the given points of amplitudes and phases.

        args:
            points ([Point]) : Array of given points
            axis ([1, 2]) : along which axis the interpolation will be done
                            1 : x axis
                            2 : y axis
            sort (boolean) : if sorting of the points is needed.
                            Set to True if the points are not structured
                            Set to False if the points are structured
    '''

    def __init__(self, points, axis=1, sort=True):
        self.points = points

        if sort:
            self.points.sort()

        self.axis = axis
        self.alpha = np.nan
        self.beta = np.nan

        if axis==1:
            # the line is along the x axis
            self.x = np.array([point.x for point in self.points])
        else:
            # the line is along the y axis
            self.x = np.array([point.y for point in self.points])

        self.a = np.array([point.a for point in self.points])
        self.p = np.array([point.p for point in self.points])

        # Check for length and unique values
        if len(self.x) < 2:
            print('Interpolant needs at least two input points.')
        else:
            if np.any(np.diff(self.x) == 0):
                print('The input must be unique elments')
            else:
                pass

    def findindices(self, point):
        __point = point

        if self.axis==1:
            __xi = __point.x
        else:
            __xi = __point.y

        __i1 = np.argmax([__xi <= __i for __i in self.x]) - 1
        __i2 = np.argmin([__xi >= __i for __i in self.x])

        return([__i1, __i2])

    def genweight(self, point, indices):
        __point = point

        if self.axis==1:
            __xi = __point.x
        else:
            __xi = __point.y

        __i1 = indices[0]
        __i2 = indices[1]

        __x1 = self.x[__i1]
        __x2 = self.x[__i2]
        __dx = __x2 - __x1

        __nantest = np.isnan(np.array([self.a[__i1], self.a[__i2]]))

        if np.all(__nantest):
            print('NaN value found around [x, y] = ', [__point.x, __point.y])
            print('Output will be nan values...')
            __alpha = np.nan
            __beta = np.nan

        elif np.any(__nantest):
            print('NaN value found for [x, y] = ', [__point.x, __point.y])
            print('Output will be nan values...')

            if np.array(np.where(__nantest)).flat[0]:
                # First element is available
                __alpha = 1
                __beta = 0
            else:
                # Second element is available
                __alpha = 0
                __beta = 1
        else:
            # Both elements are available
            if __dx == 0:
                __alpha = 1
            else:
                __alpha = (__x2 - __xi)/float(__dx)

            __beta = 1 - __alpha

        return([__alpha, __beta])

    def interpolate(self, point):
        __point = point
        __indices = self.findindices(point=__point)
        __alpha, __beta = self.genweight(point=__point, indices=__indices)

        __point1 = self.points[__indices[0]]
        __point2 = self.points[__indices[1]]

        __sinval = __alpha*__point1.a*np.sin(__point1.p)+__beta*__point2.a*np.sin(__point2.p)
        __cosval = __alpha*__point1.a*np.cos(__point1.p)+__beta*__point2.a*np.cos(__point2.p)

        __point.p = np.arctan2(__sinval, __cosval)
        __point.a = __sinval/np.sin(__point.p)

        return(__point)

class Interpolator2D(object):
    '''
    Interpolator2D(grid) create the 2D interpolator from the given grid.

    args:
        grid (Grid) :   Grid object containing amplitude and phase from which
                        the interpolation will be made
    '''
    def __init__(self, grid):
        self.sourcegrid = grid

    def interpolatepoint(self, point):
        __point = point
        __pointx = [Point(x=__point.x, y=self.sourcegrid.y[i]) for i in np.arange(self.sourcegrid.shape[0])]

        for i in np.arange(self.sourcegrid.shape[0]):
            # Finding all the interpolated points along y axis
            __points = self.sourcegrid.points[i, :]
            __interpolator = Interpolator1D(points=__points, axis=1, sort=False)
            __pointx[i] = __interpolator.interpolate(point=__pointx[i])

        __interpolator = Interpolator1D(points=__pointx, axis=2, sort=False)
        __point = __interpolator.interpolate(point=__point)

        return(__point)

    def interpolategrid(self, grid):
        __grid = grid
        __points = np.array([self.interpolatepoint(point) for point in __grid.points.flat])
        __grid.points = np.reshape(__points, __grid.shape)

        return(__grid)

if __name__=='__main__':
    # Implementation of the test cases from the original paper
    # Linear Interpolation
    point1 = Point(x=0, a=1.5, p=45)
    point2 = Point(x=1, a=2, p=-45)
    pointi = Point(x=0.65)

    Interp1D = Interpolator1D(points=[point1, point2])
    pointi = Interp1D.interpolate(pointi)

    print('{:=>40}\n{: ^40}\n{:=>40}'.format('', '1D Interpolation', ''))
    print('|{:-<38}|'.format(''))
    print('|{:^12}|{:^12}|{:^12}|'.format('Value', 'Paper', 'Script'))
    print('|{:-<38}|'.format(''))
    print('|{:^12s}|{:^12.4f}|{:^12.4f}|'.format('Amplitude', 1.4020, pointi.a))
    print('|{:^12s}|{:^12.4f}|{:^12.4f}|'.format('Phase', -0.4016, pointi.p))
    print('|{:-<38}|\n'.format(''))

    # Grid Interpolation
    # Input grid and amplitude phase
    # The amplitude and phase are meant to be input as row major format
    inx = np.arange(0, 5)
    iny = np.arange(0, 5)
    inA = np.array([[0.1526, 0.6104, 1.1075, 1.0474, 0.4655],
        [0.2135, 0.9765, 1.4736, 1.4135, 0.8315],
        [0.5000, 1.2630, 1.7601, 1.7000, 1.1180],
        [0.6634, 1.4264, 1.9234, 1.8634, 1.2814],
        [0.6787, 1.4417, 1.9387, 1.8787, 1.2967]])
    inA = np.transpose(inA) # Chaning to row major format
    inP = np.array([[6.5937, 17.4027, 122.1608, 259.5022, 348.9741],
        [342.2851, 353.0941, 97.8522, 235.1953, 324.6654],
        [294.6112, 305.4202, 50.1783, 187.5196, 276.9915,],
        [230.8300, 241.6390, 346.3971, 123.7385, 213.2103],
        [160.6516, 171.4606, 276.2187, 53.5601, 143.0320]])
    inP = np.transpose(inP) # Changing to row major format

    ingrid = Grid(x=inx, y=iny, A=inA, P=inP, isradians=False)

    # Output grid
    outx = np.arange(0.5, 4.5)
    outy = np.arange(0.5, 4.5)
    outgrid = Grid(x=outx, y=outy)

    ipl = Interpolator2D(ingrid)
    out = ipl.interpolategrid(grid=outgrid)
    print('{:=>40}\n{: ^40}\n{:=>40}'.format('', '2D Interpolation', ''))
    out.print()


"""Tests for ``siptools.scripts.create_addml`` module"""
from __future__ import unicode_literals

import os

import pytest

import lxml.etree as ET
import siptools.scripts.create_addml as create_addml
from siptools.utils import decode_path, read_md_references

CSV_FILE = "tests/data/csvfile.csv"
DELIMITER = ";"
CHARSET = "UTF-8"
RECORDSEPARATOR = "CR+LF"
QUOTINGCHAR = '"'

ADDML_NS = './/{http://www.arkivverket.no/standarder/addml}'

@pytest.mark.parametrize('is_header', [False, True])
def test_create_addml(is_header):
    """Test that ``create_addml`` returns valid addml."""

    addml_etree = create_addml.create_addml_metadata(
        csv_file=CSV_FILE, delimiter=DELIMITER, isheader=is_header,
        charset=CHARSET, record_separator=RECORDSEPARATOR,
        quoting_char=QUOTINGCHAR
    )

    # Check namespace
    assert addml_etree.nsmap['addml'] == ('http://www.arkivverket.no/'
                                          'standarder/addml')

    # Check schema
    assert addml_etree.get(
        '{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'
    ) == ('http://www.arkivverket.no/standarder/addml '
          'http://schema.arkivverket.no/ADDML/latest/addml.xsd')

    # Check individual elements
    tags = ["fieldSeparatingChar", "charset", "recordSeparator", "quotingChar"]
    vals = [";", "UTF-8", "CR+LF", '"']

    for i, tag in enumerate(tags):
        val = addml_etree.find(ADDML_NS + tag).text
        assert val == vals[i]

    # Check that flatFile element is not added
    assert addml_etree.find(ADDML_NS + "flatFile") is None

    # Check that correct amount of headers is created
    assert len(addml_etree.find(ADDML_NS + "fieldDefinitions")) == 3

@pytest.mark.parametrize('is_header', [False, True])
def test_create_addml_with_flatfile(is_header):
    """Tests that ``create_addml`` adds flatFile element if optional
    parameter flatfile_name is provided.
    """

    addml_etree = create_addml.create_addml_metadata(
        csv_file=CSV_FILE, delimiter=DELIMITER, isheader=is_header,
        charset=CHARSET, record_separator=RECORDSEPARATOR,
        quoting_char=QUOTINGCHAR, flatfile_name="path/to/test"
    )

    # Check that URL encoded path is written to flatFile element
    flatfile = addml_etree.find(ADDML_NS + "flatFile")
    assert decode_path(flatfile.get("name")) == "path/to/test"

@pytest.mark.parametrize(
    'isheader, exp_amd_files, exp_fields',
    [(False,
      ['ec816a14242f3984e483fa23174881d5-ADDML-amd.xml',
       'dd678fd96b655fd95efbb9fe4a77483a-ADDML-amd.xml'],
      [['header1', 'header2', 'header3'], ['header1', 'header2', 'header3']]),
     (True,
      ['f2d98110001385875d56ef940394f826-ADDML-amd.xml',
       'c27d34506bd21076849458c6214095c9-ADDML-amd.xml'],
      [['1', '2', '3'], ['test', 'test', 'test']])]
)
def test_create_addml_creator(testpath, isheader, exp_amd_files, exp_fields):
    """
    Test that ``create_addml`` creates addml files and
    md-reference file without unnecessary duplication.
    """
    # Common expectations
    exp_csvs = [
        ['tests/data/simple_csv.csv', 'tests/data/simple_csv_2.csv'],
        ['tests/data/csvfile.csv']
    ]
    exp_flatfiles_child_count = [4, 3]

    _create_addml(testpath, isheader)

    # Check that md-reference and the ADDML-amd files with correct content
    # are created
    assert os.path.isfile(
        os.path.join(testpath, 'create-addml-md-references.jsonl')
    )

    for amd_file_index, exp_amd_file in enumerate(exp_amd_files):
        amd_file = os.path.join(testpath, exp_amd_file)
        assert os.path.isfile(amd_file)

        root = ET.parse(amd_file)
        flat_files = root.find(ADDML_NS + "flatFiles")

        # Verify the number of child elements in flatFiles
        assert len(flat_files) == exp_flatfiles_child_count[amd_file_index]

        # Verify that right CSV files are in flatFiles
        for index, exp_csv in enumerate(exp_csvs[amd_file_index]):
            assert decode_path(flat_files[index].get('name')) == exp_csv

        # Verify fields within flatFiles
        field_definitions = root.find(ADDML_NS + "fieldDefinitions")
        for index, field in enumerate(field_definitions):
            assert field.get('name') == exp_fields[amd_file_index][index]

@pytest.mark.parametrize("file_, base_path", [
    ('tests/data/csvfile.csv', ''),
    ('./tests/data/csvfile.csv', ''),
    ('csvfile.csv', 'tests/data'),
    ('./csvfile.csv', './tests/data'),
    ('data/csvfile.csv', 'absolute')
])
def test_paths(testpath, file_, base_path, run_cli):
    """ Test the following path arguments:
    (1) Path without base_path
    (2) Path without base bath, but with './'
    (3) Path with base path
    (4) Path with base path and with './'
    (5) Absolute base path
    """
    if 'absolute' in base_path:
        base_path = os.path.join(os.getcwd(), 'tests')

    if base_path != '':
        run_cli(create_addml.main, [
            '--delim', DELIMITER, '--charset', CHARSET,
            '--sep', RECORDSEPARATOR, '--quot', QUOTINGCHAR,
            '--workspace', testpath, '--base_path', base_path, file_])
    else:
        run_cli(create_addml.main, [
            '--delim', DELIMITER, '--charset', CHARSET,
            '--sep', RECORDSEPARATOR, '--quot', QUOTINGCHAR,
            '--workspace', testpath, file_])

    references = read_md_references(testpath,
                                    'create-addml-md-references.jsonl')
    assert os.path.normpath(file_) in references

    assert os.path.isfile(os.path.normpath(os.path.join(base_path, file_)))

@pytest.mark.parametrize("filename, charset", [
    ("tests/data/valid_utf8.csv", "UTF-8"),
    ("tests/data/valid_iso8859-15.csv", "ISO-8859-15"),
    ("tests/data/valid_utf8.csv", "UTF-8"),
    ("tests/data/valid_iso8859-15.csv", "ISO-8859-15"),
])
def test_csv_header_charset(filename, charset):
    """
    Test CSV header read with different character encodings.

    :filename: Test file
    :charset: Character encoding of test file
    """
    header = create_addml.csv_header(attributes={
        "csv_file": filename, "delimiter": ",", "charset": charset,
        "isheader": True
    })
    assert header == ["year", "br\xe4nd", "m\xf6del", "detail", "other"]

    header = create_addml.csv_header(attributes={
        "csv_file": filename, "delimiter": ",", "charset": charset,
        "isheader": False
    })
    assert header == ["header1", "header2", "header3", "header4", "header5"]

def _create_addml(testpath, isheader):
    creator = create_addml.AddmlCreator(testpath)

    # Append two csv files with same
    # metadata, but different filename
    creator.add_addml_md(attributes={
        "csv_file": "tests/data/simple_csv.csv", "delimiter": ",",
        "isheader": isheader, "charset": CHARSET,
        "record_separator": RECORDSEPARATOR,
        "quoting_char": QUOTINGCHAR
    })

    creator.add_addml_md(attributes={
        "csv_file": "tests/data/simple_csv_2.csv", "delimiter": ",",
        "isheader": isheader, "charset": CHARSET,
        "record_separator": RECORDSEPARATOR,
        "quoting_char": QUOTINGCHAR
    })

    # Append csv file with different metadata
    creator.add_addml_md(attributes={
        "csv_file": CSV_FILE, "delimiter": DELIMITER, "isheader": isheader,
        "charset": CHARSET, "record_separator": RECORDSEPARATOR,
        "quoting_char": QUOTINGCHAR
    })

    creator.write()


import argparse
from typing import Dict, List

import zmq

from pysecurecircuit import const, utils
from pysecurecircuit.circuit import GarbledCircuit, GarbledGate, GarbledKey
from pysecurecircuit.ot import ot_decrypt, ot_request
from pysecurecircuit.secure_types import _SecureInt

class Client:
    def __init__(
        self,
        client_id: int,
        host: str,
        port: int,
    ) -> None:
        """
        Initializes client object.

        Args:
            client_id: ID of the client
            host: Host
            port: Port
        """
        if client_id <= 0:
            raise Exception("Invalid ID")
        elif client_id > 1:
            # Currently supports two
            raise NotImplemented

        self.client_id = client_id
        self.host = host
        self.port = port

        # Dictionary containing mapping of wire_id to its bit value
        self.wire_inputs: Dict[int, int] = {}

    def send(self, request_type: str, payload: Dict) -> None:
        """
        Send request and payload to server.

        Args:
            request_type: Request type
            payload: Dictionary of data
        """
        self.socket.send_json(dict(request=request_type, payload=payload))

    def run(self):
        """
        Connect to server and evaluate circuit.
        """
        context = zmq.Context()
        self.socket = context.socket(zmq.REQ)
        self.socket.connect(f"tcp://{self.host}:{self.port}")
        print("[!] Client started")

        # Fetch garbled circuit and circuit's metadata
        self.fetch_garbled_circuit()

        # Evaluate circuit
        for gate in self.garbled_circuit.gates.values():
            self.evaluate_gate(gate)

        # Calculate and send circuit's output
        circuit_output = self.garbled_circuit.calculate_output()
        self.send(const.REQ_OUTPUT, circuit_output)
        self.socket.recv()

        utils.print_output(circuit_output)

    def fetch_garbled_circuit(self):
        """
        Fetch garbled circuit from server.
        """
        self.send(const.REQ_FETCH_GARBLED_TABLE, {})
        data = self.socket.recv_json()

        print(f"[!] Received garbled circuit '{data['name']}'")
        # Take input from user
        self.wire_inputs = utils.take_user_input(data["inputs"])

        # Create garbled circuit object
        self.garbled_circuit = GarbledCircuit(
            data["garbled_table"], data["outputs"], data["const_keys"]
        )

        # Create graph of gates and their prerequisites
        gate_edges: List[List[int]] = data["gate_prerequisites"]

        self.circuit_graph: Dict[int, List[int]] = {}

        for edge in gate_edges:
            parent = edge[-1]
            nodes = edge[:-1]

            self.circuit_graph[parent] = nodes

    def evaluate_gate(self, gate: GarbledGate):
        """
        Evaluates garbled gate
        """

        def get_key(node) -> str:
            if node in self.garbled_circuit.gates:
                return self.garbled_circuit.gates[node].output_key.decode()
            return self.garbled_circuit.constant_keys[node]

        # If gate has no prerequisites, then fetch keys using oblivious transfer
        if gate.id not in self.circuit_graph:
            self.fetch_gate_keys(gate)
        else:
            # If gate has prerequisites, then its parent gates are already
            # evaluated and take keys from parent gates
            key0, key1 = [get_key(node) for node in self.circuit_graph[gate.id]]

            gate.input_keys.append(GarbledKey(key=key0))
            gate.input_keys.append(GarbledKey(key=key1))

        gate.evaluate()

    def fetch_key_ot(self, key: GarbledKey):
        wire_value = self.wire_inputs[key.wire_id]

        private_key, public_keys = ot_request(wire_value)

        self.send(
            const.REQ_OT_KEY_TRANSFER,
            dict(
                # input_bit=self.wire_inputs[key.wire_id],
                public_keys=public_keys,
                wire_id=key.wire_id,
                party_id=key.party_id,
            ),
        )

        data = self.socket.recv_json()

        encrypted_values = data["encrypted_values"]

        key.key = ot_decrypt(wire_value, encrypted_values, private_key)
        # key.key = data["key"]

    def fetch_gate_keys(self, gate: GarbledGate):
        self.send(const.REQ_FETCH_GARBLED_GATE_INPUT_KEYS, dict(gate_id=gate.id))
        data = self.socket.recv_json()

        keys_info: List[Dict] = data["keys_info"]

        for key_info in keys_info:
            wire_id = key_info.get("wire_id")
            key = key_info.get("key")

            if not key and wire_id in self.garbled_circuit.gates:
                key = self.garbled_circuit.gates[wire_id].output_key.decode()
            elif not key and wire_id and wire_id in self.garbled_circuit.constant_keys:
                key = self.garbled_circuit.constant_keys[wire_id]

            key = GarbledKey(
                key=key,
                wire_id=wire_id,
                party_id=key_info.get("party_id"),
            )

            gate.input_keys.append(key)
            if key.key is None:
                self.fetch_key_ot(key)

    def _take_int_input(self, input_info) -> None:
        # value = const.BOB_INPUT
        value = int(input(f"[+] Enter value for '{input_info['name']}': "))
        if value < 0:
            raise NotImplemented

        bit_str = bin(value)[2:]

        if len(bit_str) > _SecureInt.num_wires:
            raise Exception("Integer overflow")
        elif len(bit_str) < _SecureInt.num_wires:
            bit_str = "0" * (_SecureInt.num_wires - len(bit_str)) + bit_str

        self.wire_inputs.update(dict(zip(input_info["wires"], map(int, bit_str))))

    def take_user_input(self, inputs: List[Dict]):
        for input_info in inputs:
            kind = input_info["kind"]

            if kind == _SecureInt.kind:
                self._take_int_input(input_info)
            elif kind == "wire":
                self.wire_inputs[input_info["wires"][1]] = int(const.BOB_INPUT)
            else:
                raise NotImplemented