# -*- coding:utf-8 -*-
# Created by LuoJie at 11/23/19

from utils.config import save_wv_model_path, vocab_path
import tensorflow as tf
from utils.gpu_utils import config_gpu
from tensorflow.keras.models import Model
import tensorflow as tf
from utils.wv_loader import load_embedding_matrix, Vocab

class Encoder(tf.keras.Model):
    def __init__(self,  embedding_matrix,enc_units, batch_sz):
        super(Encoder, self).__init__()
        self.batch_sz = batch_sz
        self.enc_units = enc_units
        vocab_size, embedding_dim =  embedding_matrix.shape
        self.embedding =tf.keras.layers.Embedding(vocab_size, embedding_dim,
                                                  trainable = True)

        self.lstm = tf.keras.layers.LSTM(self.enc_units,
                                       return_sequences=True,
                                       return_state=True,
                                       recurrent_initializer='glorot_uniform')
        self.bilstm = tf.keras.layers.Bidirectional(self.lstm)
        #self.W_h = tf.keras.layers.Dense(self.enc_units*2)
        #默认是concat
    def call(self, x):
        # code
        x = self.embedding(x)
        #output shape = batch, max_enc_len, 2*hidden_dim
        output, forward_state,f_cell_state, backward_state,b_cell_state = self.bilstm(x)
        #print("cell shape is {}".format(cell_state.shape))
        hidden_state = tf.keras.layers.concatenate([forward_state, backward_state], axis=-1)
        cell_state = tf.keras.layers.concatenate([f_cell_state, b_cell_state], axis=-1)
        state = [hidden_state, cell_state]
        #enc_features shape == (batch_size*max_len, 2*enc_units)
        #enc_features = tf.reshape(output, (-1,2*self.enc_units))
        #enc_features = self.W_h(enc_features)
        #output= self.bilstm(x)

        return output, state

    # def initialize_hidden_state(self):
    #     return tf.zeros(shape = [self.batch_sz, self.enc_units])
class ReduceState(tf.keras.Model):
    def __init__(self,enc_units,):
        super(ReduceState, self).__init__()
        self.reduce_h = tf.keras.layers.Dense(enc_units, activation = tf.keras.activations.relu)
            #nn.Linear(config.hidden_dim * 2, config.hidden_dim)
        self.reduce_c = tf.keras.layers.Dense(enc_units,activation =tf.keras.activations.relu)
            #nn.Linear(config.hidden_dim * 2, config.hidden_dim)

    def call(self, state):
        # h, c dim = [ batch, 2*enc_units]
        h, c = state

        hidden_reduced_h = self.reduce_h(h)

        hidden_reduced_c = self.reduce_c(c)

        # h, c dim = [1, batch, hidden_dim]
        return hidden_reduced_h, hidden_reduced_c

def masked_attention(enc_padding_mask, score):
    """Take softmax of e then apply enc_padding_mask and re-normalize"""
    #attn_dist shape = batch_size, max_len,1
    attn_dist = tf.squeeze(score, axis=2)
    attn_dist = tf.nn.softmax(attn_dist) #shape = (batch_size, max_len)
    mask = tf.cast(enc_padding_mask, dtype=attn_dist.dtype)
    attn_dist *= mask  # apply mask
    masked_sums = tf.reduce_sum(attn_dist, axis=1)  # shape (batch_size)
    attn_dist = attn_dist / tf.reshape(masked_sums, [-1, 1])  # re-normalize
    attn_dist = tf.expand_dims(attn_dist, axis=2)
    return attn_dist #shape batch_size, max_len,1

class BahdanauAttention(tf.keras.layers.Layer):
    def __init__(self, units):
        super(BahdanauAttention, self).__init__()
        self.W_s = tf.keras.layers.Dense(2*units)
        self.W_h = tf.keras.layers.Dense(2*units)
        self.W_c = tf.keras.layers.Dense(2*units)
        self.V = tf.keras.layers.Dense(1)

    def __call__(self, dec_hidden, enc_output, enc_pad_mask, use_coverage=False, prev_coverage=None):
        """
         calculate attention and coverage from dec_hidden enc_output and prev_coverage
         one dec_hidden(word) by one dec_hidden
         dec_hidden or query is [batch_sz, enc_unit], enc_output or values is [batch_sz, max_train_x, enc_units],
         prev_coverage is [batch_sz, max_len_x, 1]
         dec_hidden is initialized as enc_hidden, prev_coverage is initialized as None
         output context_vector [batch_sz, enc_units] attention_weights & coverage [batch_sz, max_len_x, 1]
         """
        # hidden shape == (batch_size, hidden size)
        # hidden_with_time_axis shape == (batch_size, 1, hidden size)
        # we are doing this to perform addition to calculate the score
        #dec_hidden  shape = (batch_size, 2*dec_units)
        hidden_with_time_axis = tf.expand_dims(dec_hidden, 1)

        if use_coverage and prev_coverage is not None:
            # Multiply coverage vector by w_c to get coverage_features.
            # self.W_s(values) [batch_sz, max_len, units] self.W_h(hidden_with_time_axis) [batch_sz, 1, 2*units]
            # self.W_c(prev_coverage) [batch_sz, max_len, 2*units]  score [batch_sz, max_len, 1]
            score = self.V(tf.nn.tanh(self.W_s(enc_output) + self.W_h(hidden_with_time_axis) + self.W_c(prev_coverage)))
            # attention_weights shape (batch_size, max_len, 1)

            # attention_weights shape== (batch_size, max_length, 1)
            #mask = tf.cast(enc_pad_mask, dtype=score.dtype)

            ##attention_weights = tf.nn.softmax(score, axis=1)
            attention_weights = masked_attention(enc_pad_mask, score)
            coverage = attention_weights + prev_coverage
        else:
            # score shape == (batch_size, max_length, 1)
            # we get 1 at the last axis because we are applying score to self.V
            # the shape of the tensor before applying self.V is (batch_size, max_length, units)
            # 计算注意力权重值
            score = self.V(tf.nn.tanh(
                self.W_s(enc_output) + self.W_h(hidden_with_time_axis)))

            # mask = tf.cast(enc_pad_mask, dtype=score.dtype)
            # masked_score = tf.squeeze(score, axis=-1) * mask
            # masked_score = tf.expand_dims(masked_score, axis=2)
            # attention_weights = tf.nn.softmax(masked_score, axis=1

            attention_weights = masked_attention(enc_pad_mask, score)
            if use_coverage:
                coverage = attention_weights
            else:
                coverage = []

        # # 使用注意力权重*编码器输出作为返回值，将来会作为解码器的输入
        # context_vector shape after sum == (batch_size, 2*hidden_dim)
        context_vector = attention_weights * enc_output
        context_vector = tf.reduce_sum(context_vector, axis=1)
        attention_weights = tf.squeeze(attention_weights, -1) #转换为 batch_size, max_len

        return context_vector, attention_weights, coverage
class Decoder(tf.keras.Model):
    def __init__(self, embedding_matrix, dec_units, attn_units, batch_sz, ):
        super(Decoder, self).__init__()
        self.batch_sz = batch_sz
        self.dec_units = dec_units
        vocab_size, embedding_dim = embedding_matrix.shape
        self.attention = BahdanauAttention(attn_units)
        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim, weights=[embedding_matrix],
                                                   trainable=True)

        self.lstm = tf.keras.layers.LSTM(dec_units,
                                       return_sequences=True,
                                       # whether to return the last output sequence, or the full sequence
                                       return_state=True,  # whether to return the last state in additon to output
                                       recurrent_initializer='glorot_uniform')
        self.x_context = tf.keras.layers.Dense(embedding_dim)
        self.fc1 = tf.keras.layers.Dense(dec_units)
        self.fc2 = tf.keras.layers.Dense(vocab_size, activation=tf.keras.activations.softmax)
    def __call__(self, dec_input, hidden_state, enc_output,enc_pad_mask,
                context_vector_last, use_coverage,prev_coverage):
        #hidden_state 输入lstm的hidden_state
        #context_vector 上下文词向量 batch_size,enc_units*2

        x = self.embedding(dec_input)  # x的结果shape  =(batch_size, 1,embedding_dim)
        # 与 attention合并
        x = tf.concat([tf.expand_dims(context_vector_last, axis=1), x],
                      axis=-1)  # shape= (batch_size, 1, 2*enc_hidden_size+ embedding_dim)
        x = self.x_context(x) #重新映射为 embeding_dim

        output, hidden_state, cell_state = self.lstm(x,initial_state=hidden_state)

        # dec_state shape = 2* (batch_size,1*dec_units)
        dec_state  = [hidden_state, cell_state]
        #求本轮context_vector

        dec_state_concat = tf.concat([hidden_state, cell_state], axis =-1)
        context_vector, attentions, coverage_ret = self.attention(dec_state_concat,
                                                                  enc_output,
                                                                  enc_pad_mask,
                                                                  use_coverage,
                                                                  prev_coverage)
        #print("context vector shape", context_vector.shape)

        output = tf.reshape(output, (-1, output.shape[2]))  #batch_size, dec_units
       # 按照作者做法再与context_vector 合并  可加快训练速度
        output = tf.concat([output, context_vector], axis = -1)  #batch_size, dec_hidden_size +2*enc_units
        output = self.fc1(output)
        prediction = self.fc2(output)
        #prediction 维度 batch_size,vocab_size

        return x, prediction, dec_state,context_vector,attentions,coverage_ret

class Pointer(tf.keras.layers.Layer):

    def __init__(self):
        super(Pointer, self).__init__()
        self.w_s_reduce = tf.keras.layers.Dense(1)
        self.w_i_reduce = tf.keras.layers.Dense(1)
        self.w_c_reduce = tf.keras.layers.Dense(1)

    def call(self, context_vector, dec_hidden, dec_inp):
        #context_vector 为当前context_vector
        dec_hidden = tf.concat([dec_hidden[0], dec_hidden[1]], axis =-1)  #batch_size, 2*dec_units
        dec_inp = tf.squeeze(dec_inp, axis =1 ) #转换为 （batch_size, embedding_size)
        return tf.nn.sigmoid(self.w_s_reduce(dec_hidden) + self.w_c_reduce(context_vector) +
                             self.w_i_reduce(dec_inp))
    #shape= batch_size, 1

if __name__ == '__main__':
    # GPU资源配置
    config_gpu()
    # 读取vocab训练
    vocab = Vocab(vocab_path)
    # 计算vocab size
    vocab_size = vocab.count

    # 使用GenSim训练好的embedding matrix
    embedding_matrix = load_embedding_matrix()

    enc_max_len = 200
    dec_max_len = 41
    batch_size = 16
    embedding_dim = 300
    units = 128
    context_vector_last = tf.zeros(shape = (batch_size,2*units))
    # 编码器结构
    encoder = Encoder( embedding_matrix, units, batch_size)
    # encoder input
    enc_inp = tf.ones(shape=(batch_size, enc_max_len), dtype=tf.int32)
    # decoder input
    dec_inp = tf.ones(shape=(batch_size, dec_max_len), dtype=tf.int32)
    # enc pad mask
    enc_pad_mask = tf.ones(shape=(batch_size, enc_max_len), dtype=tf.int32)

    # encoder hidden
   # enc_hidden = encoder.initialize_hidden_state()

    enc_output, enc_hidden = encoder(enc_inp)
    # 打印结果
    print('Encoder output shape: (batch size, sequence length, units) {}'.format(enc_output.shape))
    print('Encoder Hidden state shape: (batch size, units) {}'.format(enc_hidden[0].shape))
    #
    reduce_state = ReduceState(units)
    enc_hidden = reduce_state(enc_hidden)
    # attention_layer = BahdanauAttention(units)
    # context_vector, attention_weights, coverage = attention_layer(enc_hidden, enc_output, enc_pad_mask)
    #
    # print("Attention context_vector shape: (batch size, units) {}".format(context_vector.shape))
    # print("Attention weights shape: (batch_size, sequence_length, 1) {}".format(attention_weights.shape))
    # print("Attention coverage: (batch_size, ) {}".format(coverage))

    decoder = Decoder( embedding_matrix, units,units, batch_size)

    dec_x, dec_out, dec_hidden, context_vector, attentions, coverage_ret= decoder(tf.random.uniform((batch_size, 1)),
                                          enc_hidden,enc_output,enc_pad_mask,
                                          context_vector_last, True, None)
    print('Decoder output shape: (batch_size, vocab size) {}'.format(dec_out.shape))
    print('Decoder dec_x shape: (batch_size, 1,embedding_dim + units) {}'.format(dec_x.shape))
    print("decoder dec_hidden shape:{}".format(dec_hidden[0].shape))
    print("context_vector shape:{}".format(context_vector.shape))
    pointer = Pointer()
    p_gen = pointer(context_vector, dec_hidden, dec_x)
    print('Pointer p_gen shape: (batch_size,1) {}'.format(p_gen.shape))


from unittest.mock import patch
from rest_framework import status
from users.tests.factories import UserFactory
from subscriptions.vendor.apple.apple import AppleNotificationHandler
from payments.tests.factories import PaymentTransactionFactory, PaymentMethodFactory
from subscriptions.tests.factories import SubscriptionFactory, SubscriptionPlanFactory
from amuse.tests.test_api.base import AmuseAPITestCase

class TestChangeRenewalHandler(AmuseAPITestCase):
    def setUp(self):
        self.test_payload = {
            'latest_receipt': 'a',
            'auto_renew_product_id': 'amuse_pro_monthly_renewal',
            'auto_renew_status_change_date_pst': '2021-02-17 20:47:06 America/Los_Angeles',
            'unified_receipt': {
                'latest_receipt': 'a',
                'pending_renewal_info': [
                    {
                        'original_transaction_id': '430000729594340',
                        'product_id': 'amuse_pro_monthly_renewal',
                        'auto_renew_status': '1',
                        'auto_renew_product_id': 'amuse_pro_monthly_renewal',
                    }
                ],
                'environment': 'Production',
                'status': 0,
                'latest_receipt_info': [
                    {
                        'expires_date_pst': '2021-03-17 20:47:02 America/Los_Angeles',
                        'purchase_date': '2021-02-18 04:47:02 Etc/GMT',
                        'in_app_ownership_type': 'PURCHASED',
                        'purchase_date_ms': '1613623622000',
                        'original_purchase_date_ms': '1605669814000',
                        'transaction_id': '430000787887990',
                        'original_transaction_id': '430000729594340',
                        'quantity': '1',
                        'expires_date_ms': '1616039222000',
                        'original_purchase_date_pst': '2020-11-17 19:23:34 America/Los_Angeles',
                        'product_id': 'amuse_pro_monthly_renewal',
                        'subscription_group_identifier': '20581044',
                        'web_order_line_item_id': '430000302577640',
                        'expires_date': '2021-03-18 03:47:02 Etc/GMT',
                        'is_in_intro_offer_period': 'false',
                        'original_purchase_date': '2020-11-18 03:23:34 Etc/GMT',
                        'purchase_date_pst': '2021-02-17 20:47:02 America/Los_Angeles',
                        'is_trial_period': 'false',
                    },
                    {
                        'expires_date_pst': '2021-01-17 19:23:32 America/Los_Angeles',
                        'purchase_date': '2020-12-18 03:23:32 Etc/GMT',
                        'in_app_ownership_type': 'PURCHASED',
                        'purchase_date_ms': '1608261812000',
                        'original_purchase_date_ms': '1605669814000',
                        'transaction_id': '430000745670487',
                        'original_transaction_id': '430000729594340',
                        'quantity': '1',
                        'expires_date_ms': '1610940212000',
                        'original_purchase_date_pst': '2020-11-17 19:23:34 America/Los_Angeles',
                        'product_id': 'amuse_pro_monthly_renewal',
                        'subscription_group_identifier': '20581044',
                        'web_order_line_item_id': '430000293325421',
                        'expires_date': '2021-01-18 03:23:32 Etc/GMT',
                        'is_in_intro_offer_period': 'false',
                        'original_purchase_date': '2020-11-18 03:23:34 Etc/GMT',
                        'purchase_date_pst': '2020-12-17 19:23:32 America/Los_Angeles',
                        'is_trial_period': 'false',
                    },
                    {
                        'expires_date_pst': '2020-12-17 19:23:32 America/Los_Angeles',
                        'purchase_date': '2020-11-18 03:23:32 Etc/GMT',
                        'in_app_ownership_type': 'PURCHASED',
                        'purchase_date_ms': '1605669812000',
                        'original_purchase_date_ms': '1605669814000',
                        'transaction_id': '430000729594340',
                        'original_transaction_id': '430000729594340',
                        'quantity': '1',
                        'expires_date_ms': '1608261812000',
                        'original_purchase_date_pst': '2020-11-17 19:23:34 America/Los_Angeles',
                        'product_id': 'amuse_pro_monthly_renewal',
                        'subscription_group_identifier': '20581044',
                        'web_order_line_item_id': '430000293325420',
                        'expires_date': '2020-12-18 03:23:32 Etc/GMT',
                        'is_in_intro_offer_period': 'false',
                        'original_purchase_date': '2020-11-18 03:23:34 Etc/GMT',
                        'purchase_date_pst': '2020-11-17 19:23:32 America/Los_Angeles',
                        'is_trial_period': 'false',
                    },
                ],
            },
            'auto_renew_status_change_date_ms': '1613623626000',
            'latest_receipt_info': {
                'original_purchase_date_pst': '2020-11-17 19:23:34 America/Los_Angeles',
                'quantity': '1',
                'subscription_group_identifier': '20581044',
                'unique_vendor_identifier': 'EC0C1741-86E6-4F1D-A10E-CA1591D91845',
                'original_purchase_date_ms': '1605669814000',
                'expires_date_formatted': '2021-03-18 03:47:02 Etc/GMT',
                'is_in_intro_offer_period': 'false',
                'purchase_date_ms': '1613623622000',
                'expires_date_formatted_pst': '2021-03-17 20:47:02 America/Los_Angeles',
                'is_trial_period': 'false',
                'item_id': '1491033244',
                'unique_identifier': '00008030-001C24C621DB802E',
                'original_transaction_id': '430000729594340',
                'expires_date': '1616039222000',
                'app_item_id': '1160922922',
                'transaction_id': '430000787887990',
                'in_app_ownership_type': 'PURCHASED',
                'bvrs': '2718',
                'web_order_line_item_id': '430000302577640',
                'version_external_identifier': '840349842',
                'bid': 'io.amuse.ios',
                'product_id': 'amuse_pro_monthly_renewal',
                'purchase_date': '2021-02-18 04:47:02 Etc/GMT',
                'purchase_date_pst': '2021-02-17 20:47:02 America/Los_Angeles',
                'original_purchase_date': '2020-11-18 03:23:34 Etc/GMT',
            },
            'notification_type': 'DID_CHANGE_RENEWAL_STATUS',
            'auto_renew_status_change_date': '2021-02-18 04:47:06 Etc/GMT',
            'environment': 'PROD',
            'auto_renew_status': 'true',
            'bvrs': '2400',
            'bid': 'io.amuse.ios',
        }

        self.user = UserFactory()
        self.payment_method = PaymentMethodFactory(
            external_recurring_id='430000729594340', method='AAPL', user=self.user
        )
        self.plan = self.plan = SubscriptionPlanFactory(
            apple_product_id='amuse_pro_monthly_renewal', trial_days=0
        )
        self.sub = SubscriptionFactory(plan=self.plan, user=self.user)
        self.payment = PaymentTransactionFactory(
            user=self.user,
            plan=self.plan,
            subscription=self.sub,
            external_transaction_id='430000729594340',
        )
        self.user = self.payment.user
        self.handler = AppleNotificationHandler()

    @patch('subscriptions.vendor.apple.apple.logger')
    def test_payload_validator(self, mock_logger):
        is_valid = self.handler.is_payload_valid(self.test_payload)
        assert is_valid == True
        invalid_data = self.test_payload
        del invalid_data['unified_receipt']
        not_valid = self.handler.is_payload_valid(invalid_data)
        assert not_valid == False
        self.assertTrue(mock_logger.warning.called)

    @patch('subscriptions.vendor.apple.change_renewal_handler.logger')
    def test_auto_renew_enabled(self, mock_logger):
        response = self.handler.process_notification(self.test_payload)
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertTrue(mock_logger.info.called)
        self.assertTrue(mock_logger.warning.not_called)

    @patch('subscriptions.vendor.apple.change_renewal_handler.logger')
    def test_auto_renew_disabled(self, mock_logger):
        new_payload = self.test_payload
        new_payload['unified_receipt']['pending_renewal_info'][0][
            'auto_renew_status'
        ] = '0'
        response = self.handler.process_notification(new_payload)
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.sub.refresh_from_db()
        self.assertEqual(self.sub.valid_until, self.payment.paid_until.date())
        self.assertTrue(mock_logger.info.called)
        self.assertTrue(mock_logger.warning.not_called)

    @patch('subscriptions.vendor.apple.change_renewal_handler.logger')
    def test_tx_not_found(self, mock_logger):
        self.payment.external_transaction_id = 'a'
        self.payment.save()
        response = self.handler.process_notification(self.test_payload)
        self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
        self.assertTrue(mock_logger.warning.called)


import json
from backend.request import *

import time

def my_seat(cookie, is_print=True):
    if is_print:
        print("请稍候...")
    url = "http://office.chaoxing.com/data/apps/seat/index?mappId=0"

    response = get_req(url, cookie)
    seats = json.loads(response.text)["data"]["curReserves"]

    res = []
    print("+{}+{}+{}+{}+{}+{}+{}+".format("-" * 4, "-" * 13, "-" * 27, "-" * 6, "-" * 8, "-" * 21, "-" * 21))
    print("|{:^4}|{:^13}|{:^27}|{:^6}|{:^8}|{:^21}|{:^21}|".format("no", "id", "floor", "seat", "status", "startTime", "endTime"))
    print("+{}+{}+{}+{}+{}+{}+{}+".format("-" * 4, "-" * 13, "-" * 27, "-" * 6, "-" * 8, "-" * 21, "-" * 21))
    for i, one_seat in enumerate(seats):
        res.append({
            "id": str(one_seat["id"]),
            "roomId": str(one_seat["roomId"]),
            "seatNum": one_seat["seatNum"],
            "status": str(one_seat["status"])
        })

        if is_print:
            print("|{:^4}|{:^13}|{:^20}|{:^6}|{:^8}|{:^21}|{:^21}|".format(str(i), str(one_seat["id"]), one_seat["firstLevelName"] + one_seat["secondLevelName"] + one_seat["thirdLevelName"], one_seat["seatNum"], str(one_seat["status"]), convert_time(one_seat["startTime"]/1000), convert_time(one_seat["endTime"]/1000)))

    print("+{}+{}+{}+{}+{}+{}+{}+".format("-" * 5, "-" * 13, "-" * 31, "-" * 7, "-" * 7, "-" * 21, "-" * 21))
    return res

def convert_time(timestamp):
    time_local = time.localtime(timestamp)
    return time.strftime("%Y-%m-%d %H:%M:%S", time_local)


import json

def combine_json_members(input_file):
    with open(input_file) as file:
        data = json.load(file)

    combined_member = {}
    print(data['2505-Lommel Barrier'])
    for member in data:
        for key, value in data[member]['Attributen']['Mobiliteitsaanbod'].items():
            print(key)
            print(value)

            combined_member[key] = combined_member.get(key, value)

    return combined_member

def write_output_file(output_file, data):
    with open(output_file, 'w') as file:
        json.dump(data, file, indent=4)

# Usage example
input_file = 'input.json'
output_file = 'output.json'

combined_member = combine_json_members(input_file)
write_output_file(output_file, combined_member)


import torch
import os
import sklearn
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from datetime import datetime
from src.definitions import DOCUMENTATION_ROOT, DATA_ROOT
from os import path

def get_documentation_root_subdir(subdir: str, filename: str) -> 'os.path':

    """
    Given a specified sub-directory and a filename, will greate the subdirectory if it does not exist.

    :param subdir: the name of the wanted subdirectory
    :param filename: the name of the filename you want under the subdirectory.
    :return: the path to the file within the subdirectory
    """

    sub_dir_path = path.join(DOCUMENTATION_ROOT, subdir)
    if not path.isdir(sub_dir_path):
        os.mkdir(sub_dir_path)

    return path.join(DOCUMENTATION_ROOT, subdir+"/"+filename)

def output_to_csv(test_scaler: 'sklearn.preprocessing.minmax_scale', model: str, filename: str,
                  predicted_output: 'np.array', test_data: 'np.array'):
    """
    Will write rescaled predicted and test data to a csv for later profit-evaluation

    :param test_scaler: the scaler to turn normalized prices back to USD.
    :param model: The name of the model where the data was predicted
    :param filename: the filename of the dataset used for testing/training
    :param predicted_output: the predicted normalized data as a np.array
    :param test_data: the predicted normalized data as a np.array
    :return: None
    """

    predicted_output = test_scaler.inverse_transform(predicted_output)
    test_data = test_scaler.inverse_transform(test_data)
    if not os.path.exists(DATA_ROOT):
        os.mkdir(DATA_ROOT)

    today = datetime.today().strftime('%Y-%m-%d')
    file = filename.split('_')[0] + '_' + model + '_' + 'output' '_' + today
    data_path = os.path.join(DATA_ROOT, 'stored_outputs/')
    pred_df = pd.DataFrame(zip(predicted_output.reshape(-1), test_data.reshape(-1)))
    complete_filename = data_path + file

    if not os.path.exists(complete_filename):
        pred_df.to_csv(complete_filename, index=False)

def get_data_from_csv(filename: str, included_features: 'np.array', data_points: int):
    """
    Get wanted data from a given csv

    :param filename: The CSV to read the files from
    :param included_features: What features you want included in the dataset
    :param data_points: Number of datapoints you want in the set, will take x most recent ones
    :return: (selected_attribute, time_index)
    """
    data = pd.read_csv(filename)
    close_price = data[included_features].tail(data_points)
    return close_price, pd.to_datetime(data["Date"])

def plot_result(model_type, y_pred: 'np.array', y: 'np.array', times: 'np.datetime_data', filename: str,
                accuracy: float, save=False):
    """
    Plot and show accuracy on trained model

    :param model_type: name of the model where the values were predicted
    :param y_pred: predicted output
    :param y: expected output
    :param times: the time-index of the test-data
    :param filename: the dataset used
    :param accuracy: the accuracy of the model
    :param save: Whether or not to save the plot under documentation/plots
    :return: None
    """
    stockname = filename.split("_")[0]
    fig = plt.figure(figsize=(8, 4))
    plt.xlabel(model_type + ' on ' + stockname + ", accuracy: " + str(accuracy))
    plt.ylabel('Normalized price')

    plt.plot(times, y, label='Actual', c='blue')
    plt.plot(times, y_pred, label='TrainedPoints', c='green')
    plt.legend()
    try:
        # plt.show()
        pass
    except Exception:
        save = True
    if save:
        img_name = "{}_{}_{}.png".format(model_type, stockname, accuracy)
        img_path = get_documentation_root_subdir("plots", img_name)
        fig.savefig(img_path)

def plot_loss(model_type: str, loss_list: 'np.array', filename: str, accuracy: float, save=False):

    """
    Plot the loss of a given model

    :param model_type: the type of model trained
    :param loss_list: the losses at the different epochs
    :param filename: the filname of the dataset used for training
    :param accuracy: final accuracy of the model
    :param save: save the model under documentation/plots
    :return: None
    """

    stockname = filename.split("_")[0]
    fig = plt.figure(figsize=(8, 4))
    plt.xlabel(model_type + ' on ' + stockname + ", accuracy: " + str(accuracy))
    plt.ylabel('Loss')

    plt.plot(range(len(loss_list)), loss_list, label='loss', c='green')
    plt.legend()
    try:
        # plt.show()
        pass
    except Exception:
        save = True
    if save:
        img_name = "{}_LOSS_{}_{}.png".format(model_type, stockname, accuracy)
        img_path = get_documentation_root_subdir("plots", img_name)
        fig.savefig(img_path)

def save_model(model_type: str, model: 'torch.nn.Module', data_filename: str, accuracy: float):
    """
    Save a trained model

    :param model_type: the type of model
    :param model: the model object
    :param data_filename: the filename of the dataset trained on
    :param accuracy: the final accuracy of the model
    :return: None
    """
    instrument = data_filename.split("_")[0]
    acc_string = str(accuracy).split(".")[1]
    model_filename = "{}_model_{}_{}.bin".format(model_type, instrument, acc_string)
    torch.save(model.state_dict(), get_documentation_root_subdir("generated_models", model_filename))


import open3d.ml.torch as ml3d
import open3d.ml as _ml3d

class C_KhSS:

    def __init__(self) -> None:
        pass

    def SettingSS(self,ModelName,ModelConfigPath):
        self.ModelName = ModelName
        self.ModelConfigPath = ModelConfigPath + self.ModelName + '.yml'
        self.ModelConfig = _ml3d.utils.Config.load_from_file(self.ModelConfigPath)

        if(self.ModelName == 'RandLANet'):
            self.RandLANetModel = ml3d.models.RandLANet(**self.ModelConfig.model)
            self.SSPipeline = ml3d.pipelines.SemanticSegmentation(self.RandLANetModel,None,device="gpu",**self.ModelConfig.pipeline)
            self.CheckPointFilePath = ModelConfigPath + self.ModelConfig['model']['ckpt_path']
            self.SSPipeline.load_ckpt(ckpt_path=self.CheckPointFilePath)

    def Run(self,Data):
        print('Model Inference Ready')
        SSResult = self.SSPipeline.run_inference(Data)
        self.ResultData = {'name': 'kh','points': Data["point"], 'color':Data["feat"],'label':SSResult["predict_labels"]}

    def Get_ResultData(self):
        return self.ResultData


from django.http import JsonResponse
import csv
from .models import *
from .serializers import *
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework import status
from datetime import datetime, timedelta
from MBATrueSight import *
import datetime as dt
import pandas as pd
import numpy as np
from datetime import datetime
from django.db.models import Q
import numbers
from django.db.models import Avg, Count, Max

@api_view(['GET'])
def index(request):
    date = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
    message = 'server is live current time is '
    return Response(data=message + date, status = status.HTTP_200_OK)

#Countries
@api_view(['GET','POST'])
def countryList(request, format=None):

    if request.method == 'GET':
    #get all countries, serialize and return json
        countries = Country.objects.all()
        serializer = CountrySerializer(countries, many=True)
        return Response(serializer.data)

    if request.method == 'POST':
        serializer = CountrySerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status = status.HTTP_201_CREATED)

@api_view(['GET','PUT','DELETE'])
def countryDetail(request, countryId, format=None):

    try:
        country = Country.objects.get(countryId=countryId)
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    if request.method == 'GET':
        serializer = CountrySerializer(country)
        return Response(serializer.data)

    elif request.method =='PUT':
        serializer = CountrySerializer(country,data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        country.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

#UserType
@api_view(['GET','POST'])
def userTypeList(request, format=None):
    if request.method == 'GET':
        userTypes = UserType.objects.all()
        serializer = UserTypeSerializer(userTypes, many=True)
        return Response(serializer.data)

    if request.method=='POST':
        serializer=UserTypeSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status = status.HTTP_201_CREATED)

@api_view(['GET','PUT','DELETE'])
def userTypeDetail(request,userTypeId,format=None):

    try:
        userType = UserType.objects.get(userTypeId=userTypeId)
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    if request.method == 'GET':
        serializer = UserTypeSerializer(userType)
        return Response(serializer.data)

    elif request.method =='PUT':
        serializer = UserTypeSerializer(userType,data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        userType.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

#StatExportType
@api_view(['GET','POST'])
def statExportTypeList(request, format=None):
    if request.method == 'GET':
        statExportTypes = StatExportType.objects.all()
        serializer = StatExportTypeSerializer(statExportTypes, many=True)
        return Response(serializer.data)

    if request.method=='POST':
        serializer=StatExportTypeSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status = status.HTTP_201_CREATED)

@api_view(['GET','PUT','DELETE'])
def statExportTypeDetail(request,statExportTypeId,format=None):

    try:
        statExportType = StatExportType.objects.get(statExportTypeId=statExportTypeId)
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    if request.method == 'GET':
        serializer = StatExportTypeSerializer(statExportType)
        return Response(serializer.data)

    elif request.method =='PUT':
        serializer = StatExportTypeSerializer(statExportType,data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        statExportType.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

#PredictionType
@api_view(['GET','POST'])
def predictionTypeList(request, format=None):
    if request.method == 'GET':
        predictionTypes = PredictionType.objects.all()
        serializer = PredictionTypeSerializer(predictionTypes, many=True)
        return Response(serializer.data)

    if request.method=='POST':
        serializer=PredictionTypeSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status = status.HTTP_201_CREATED)

@api_view(['GET','PUT','DELETE'])
def predictionTypeDetail(request,predictionTypeId,format=None):

    try:
        predictionType = PredictionType.objects.get(predictionTypeId=predictionTypeId)
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    if request.method == 'GET':
        serializer = PredictionTypeSerializer(predictionType)
        return Response(serializer.data)

    elif request.method =='PUT':
        serializer = PredictionTypeSerializer(predictionType,data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        predictionType.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

#University
@api_view(['GET','POST'])
def universityList(request, format=None):
    if request.method == 'GET':
        universities = University.objects.all()
        serializer = UniversitySerializer(universities, many=True)
        return Response(serializer.data)

    if request.method=='POST':
        serializer=UniversitySerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status = status.HTTP_201_CREATED)

@api_view(['GET','PUT','DELETE'])
def universityDetail(request,universityId,format=None):

    try:
        university = University.objects.get(universityId=universityId)
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    if request.method == 'GET':
        serializer = UniversitySerializer(university)
        return Response(serializer.data)

    elif request.method =='PUT':
        serializer = UniversitySerializer(university,data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        university.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

#Prediction
@api_view(['GET','POST'])
def predictionList(request, format=None):
    if request.method == 'GET':
        predictions = Prediction.objects.all()
        serializer = PredictionSerializer(predictions, many=True)
        return Response(serializer.data)

    if request.method=='POST':
        serializer=PredictionSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status = status.HTTP_201_CREATED)

@api_view(['GET','PUT','DELETE'])
def predictionDetail(request,predictionId,format=None):

    try:
        prediction = Prediction.objects.get(predictionId=predictionId)
    except:
        return Response(data={"Message":"No se encontraron predicciones con ese id"},status=status.HTTP_400_BAD_REQUEST)

    if request.method == 'GET':
        serializer = PredictionSerializer(prediction)
        return Response(serializer.data)

    elif request.method =='PUT':
        serializer = PredictionSerializer(prediction,data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        prediction.delete()
        return Response(data={"Message":"Prediccion borrada exitosamente."},status=status.HTTP_200_NO_CONTENT)

#User
@api_view(['GET','POST'])
def userList(request, format=None):
    if request.method == 'GET':
        users = User.objects.all()
        serializer = UserSerializer(users, many=True)
        return Response(serializer.data)

    if request.method=='POST':
        serializer=UserSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status = status.HTTP_201_CREATED)

#User by email
@api_view(['POST'])
def getUserByEmail(request, format=None):
    try:
        email = request.data['email']
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    users = User.objects.filter(email=email)

    serializer = UserSerializer(users, many=True)
    return Response(serializer.data)

#UserInfo by email
@api_view(['POST'])
def getUserByEmail(request, format=None):
    try:
        email = request.data['email']
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    users = User.objects.filter(email=email)
    serializer = UserSerializer(users, many=True)

    finalResponse = []
    userdict, userinfodict = {},{}
    userdict['user']=serializer.data

    finalResponse.append(userdict)

    for user in serializer.data:
        try:
            userId = user['userId']
            userInfos = UserInfo.objects.filter(userId=userId)
            uiSerializer = UserInfoSerializer(userInfos,many=True)
            userinfodict['userInfo']=uiSerializer.data
            finalResponse.append(userinfodict)

        except:
            response = {"Response":"No userinfo exists with this email {email}"}
            finalResponse.append(response)

    return Response(finalResponse)

#User Info PUT
@api_view(['PUT'])
def editUserInfo(request, userId, format=None):

    try:
        userInfo = UserInfo.objects.get(userId=userId)
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    if request.method =='PUT':
        request.data['updatedDate']=datetime.now()
        serializer = UserInfoSerializer(userInfo,data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

#UserInfo
@api_view(['GET','POST'])
def userInfoList(request, format=None):
    if request.method == 'GET':
        userInfos = UserInfo.objects.all()
        serializer = UserInfoSerializer(userInfos, many=True)
        return Response(serializer.data)

    if request.method=='POST':
        serializer=UserInfoSerializer(data=request.data)

        if serializer.is_valid(raise_exception=True):
            serializer.save()
            return Response(serializer.data, status = status.HTTP_201_CREATED)

#-------------------------------
#Models
@api_view(['GET'])
def predictionTrain(request,format=None):
    try:
        run_training()
        return Response(status=status.HTTP_200_OK)
    except:
        return Reponse(status=status.HTTP_400_BAD_REQUEST)

@api_view(['POST'])
def makePrediction(request,format=None):

    predictions = []

    try:
        request.data[0]['userId']
        request.data[0]['gmat']
        request.data[0]['gpa']
        request.data[0]['wk_xp']
        request.data[0]['app_type']
    except:
        return Response(data={"Error":"Datos ingresados incompletos"},status=status.HTTP_400_BAD_REQUEST)

    for entry in request.data:
        try:
            userId = entry.pop('userId')
        except:
            userId = None
        result = RFpredict(entry)
        predictions.append(result)

    try:
        request.data[0]['userId']=userId
    except:
        print(type(request.data[0]))

    try:
        request.data[0]['predictionTypeId']=1 #simple
    except:
        print(type(request.data[0]))

    try:
        request.data[0]['studentId']=""
    except:
        pass

    try:
        request.data[0]['gmatScore']=request.data[0].pop('gmat')
        request.data[0]['gpaScore']=request.data[0].pop('gpa')
        request.data[0]['workExp']=request.data[0].pop('wk_xp')
        request.data[0]['appType']=request.data[0].pop('app_type')
        request.data[0]['gradGpaScore']=round(predictions[0][0],2)
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    serializer = PredictionSerializer(data=request.data[0])
    serializer.is_valid(raise_exception=True)
    serializer.save()
    return Response(serializer.data)

@api_view(['POST'])
def makeMassivePrediction(request,format=None):

    input_file = request.FILES['file']
    try:
        df = pd.read_csv(input_file)
    except:
         try:
            df = pd.read_excel(input_file)
         except:
             response = Response(status=status.HTTP_400_BAD_REQUEST)
             return response

    df = df.reset_index()
    headers = list(df.columns.values)
    headers = [e for e in headers if e not in ('gmat','gpa','wk_xp','app_type','index','grad_gpa','student_id')]

    finalPredictions = []
    responsePredictions = []
    for index, row in df.iterrows():
        predictionArray = []
        singlePredictionDict ={}

        try:
            gmat = float(row['gmat'])
            gpa = float(row['gpa'])
            wk_xp = float(row['wk_xp'])
            app_type = float(row['app_type'])
        except:
            continue

        try:
            student_id = str(row['student_id'])
        except:
            student_id = ""

        predictionArray.extend((gmat,gpa,wk_xp,app_type))

        result = RFpredict(predictionArray)

        singlePredictionDict = {
            "gmat":gmat,
            "gpa":gpa,
            "wk_xp":wk_xp,
            "app_type":app_type,
            "grad_gpa":result[0],
            "student_id":student_id
        }
        print(singlePredictionDict)
        responsePredictionDict = singlePredictionDict.copy()

        finalPredictions.append(singlePredictionDict)

        for head in headers:
            responsePredictionDict[head] = row[head]

        responsePredictions.append(responsePredictionDict)

    response = Response()
    response.data=responsePredictions

    print(finalPredictions)

    try:
        massivePredictionId = request.data['massive_prediction_id']
    except:
        try:
            massivePredictionId =  Prediction.objects.filter(~Q(massivePredictionId=None)).last().massivePredictionId
            if isinstance(massivePredictionId,numbers.Number):
                massivePredictionId = massivePredictionId + 1
            else:
                massivePredictionId = 1
        except:
            response = Response(status=status.HTTP_400_BAD_REQUEST)
            return response

    userId = int(request.data['user_id'])
    predictionTypeId = 2

    count=0
    for pred in finalPredictions:
        count=count+1
        print(count)
        try:
            pred['gmatScore']=pred.pop('gmat')
            pred['gpaScore']=pred.pop('gpa')
            pred['workExp']=pred.pop('wk_xp')
            pred['appType']=pred.pop('app_type')
            pred['gradGpaScore']=round(pred.pop('grad_gpa'),2)
            pred['userId']=userId
            pred['studentId']=pred.pop('student_id')
            pred['predictionTypeId']=predictionTypeId
            pred['massivePredictionId']=massivePredictionId
            serializer = PredictionSerializer(data=pred)
            serializer.is_valid(raise_exception=True)
            serializer.save()
        except:
            continue

    return (response)

@api_view(['POST'])
def predictionsByUserId(request, format=None):
    if request.method == 'POST':
        try:
            userId = request.data['userId']
        except:
            return Response(data={"Error":"Por favor, ingrese el id del usuario"},status=status.HTTP_400_BAD_REQUEST)

        predictions = Prediction.objects.filter(userId=userId)

        serializer = PredictionSerializer(predictions, many=True)
        return Response(serializer.data)

@api_view(['DELETE'])
def deletePredictionsByUserId(request,userId, format=None):

    if userId == None:
        return Response(data={"Error":"Por favor, ingrese el id del usuario"},status=status.HTTP_400_BAD_REQUEST)

    try:
        predictions = Prediction.objects.filter(userId=userId)
    except:
        return Response(data={"Error":"Por favor, ingrese el id del usuario"},status=status.HTTP_400_BAD_REQUEST)

    if not predictions:
        return Response(data={"Mensaje":"No se encontraron predicciones con este id"},status=status.HTTP_400_BAD_REQUEST)

    print(predictions)
    predictions.delete()

    return Response(data={"Mensaje":"Predicciones borradas satisfactoriamente"},status=status.HTTP_200_OK)

@api_view(['DELETE'])
def deletePredictionsByMassivePredictionId(request,massivePredictionId,format=None):

    try:
        predictions = Prediction.objects.filter(massivePredictionId=massivePredictionId)
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    predictions.delete()

    return Response(status=status.HTTP_204_NO_CONTENT)

@api_view(['DELETE'])
def deletePredictionsByMassivePredictionIdAndUserId(request,massivePredictionId,userId,format=None):

    try:
        predictions = Prediction.objects.filter(massivePredictionId=massivePredictionId, userId=userId)
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    predictions.delete()

    return Response(status=status.HTTP_204_NO_CONTENT)

@api_view(['POST'])
def getPredictionsByDate(request, format=None):

    try:
        userId = request.data['userId']
    except:
        return Response(data={"Error":"Datos ingresados incompletos"},status=status.HTTP_400_BAD_REQUEST)

    try:
        startDate = request.data['startDate']
    except:
        startDate = '01/01/1995'

    try:
        endDate = request.data['endDate']
    except:
        endDate = '12/31/2050'

    try:
        smonth, sday, syear = startDate.split("/",2)
        emonth, eday, eyear = endDate.split("/",2)
        smonth, sday, syear = int(smonth), int(sday), int(syear)
        emonth, eday, eyear = int(emonth), int(eday), int(eyear)
    except:
        startDate = '01/01/1995'
        endDate = '12/31/2050'
        smonth, sday, syear = startDate.split("/",2)
        emonth, eday, eyear = endDate.split("/",2)
        smonth, sday, syear = int(smonth), int(sday), int(syear)
        emonth, eday, eyear = int(emonth), int(eday), int(eyear)

    predictions = Prediction.objects.filter(creationDate__gte=dt.date(syear, smonth, sday),
                                        creationDate__lte=dt.date(eyear, emonth, eday),
                                        userId=userId)

    serializer = PredictionSerializer(predictions, many=True)
    return Response(serializer.data)

@api_view(['GET'])
def calculateAverage(request, format=None):
    if request.method == 'GET':
        predictions = Prediction.objects.filter(gmatScore__gte=200, gpaScore__gte=2.0)
        avgGpa = predictions.aggregate(Avg('gpaScore'))
        avgGmat = predictions.aggregate(Avg('gmatScore'))
        gradGpaAvg = predictions.aggregate(Avg('gradGpaScore'))
        workExpAvg = predictions.aggregate(Avg('workExp'))
        countOfType = Prediction.objects.raw('select 1 as prediction_id, app_type, count(app_type) as work_exp from prediction group by app_type order by count(app_type) desc')
        averageType = 2
        total = 0
        occurrencesAppType = 0
        for index, i in enumerate(countOfType):
            if index == 0:
                modeAppType = i.appType
                occurrencesAppType = i.workExp
            total += i.workExp

        percAppType = round(((occurrencesAppType*100.0)/total),2)

        gpaAvg=round(avgGpa['gpaScore__avg'],2)
        gmatAvg=round(avgGmat['gmatScore__avg'],2)
        gradGpaAvg = round(gradGpaAvg['gradGpaScore__avg'],2)
        workExpAvg = round(workExpAvg['workExp__avg'],2)
        avgWorkExp = countOfType

        predAvg = {'gmatAvg':gmatAvg, 'gpaAvg':gpaAvg, 'gradGpaAvg':gradGpaAvg, 'workExpAvg':workExpAvg,
        'appTypeAvg':modeAppType,'averageType':averageType}

        serializer = PredictionAverageValuesSerializer(data=predAvg)
        serializer.is_valid(raise_exception=True)
        serializer.save()

        serializer.data['occurrencesAppType'] = occurrencesAppType
        serializer.data['occurrencesAppTypeTotal'] = total

        return Response(serializer.data, status = status.HTTP_201_CREATED)

@api_view(['GET'])
def calculateAverageBase(request, format=None):
    if request.method == 'GET':
        predictions = PredictionTraining.objects.filter(gmatScore__gte=200, gpaScore__gte=2.0)
        avgGpa = predictions.aggregate(Avg('gpaScore'))
        avgGmat = predictions.aggregate(Avg('gmatScore'))
        gradGpaAvg = predictions.aggregate(Avg('gradGpaScore'))
        workExpAvg = predictions.aggregate(Avg('workExp'))
        countOfType = PredictionTraining.objects.raw('select 1 as prediction_training_id, app_type, count(app_type) as work_exp from prediction_training group by app_type order by count(app_type) desc')
        averageType = 1

        gpaAvg=round(avgGpa['gpaScore__avg'],2)
        gmatAvg=round(avgGmat['gmatScore__avg'],2)
        gradGpaAvg = round(gradGpaAvg['gradGpaScore__avg'],2)
        workExpAvg = round(workExpAvg['workExp__avg'],2)
        avgWorkExp = countOfType

        total = 0
        occurrencesAppType = 0
        for index, i in enumerate(countOfType):
            if index == 0:
                modeAppType = i.appType
                occurrencesAppType = i.workExp
            total += i.workExp

        percAppType = round(((occurrencesAppType*100.0)/total),2)

        predAvg = {'gmatAvg':gmatAvg, 'gpaAvg':gpaAvg, 'gradGpaAvg':gradGpaAvg, 'workExpAvg':workExpAvg,
        'appTypeAvg':modeAppType,'averageType':averageType}

        serializer = PredictionAverageValuesSerializer(data=predAvg)
        serializer.is_valid(raise_exception=True)
        serializer.save()

        serializer.data['occurrencesAppType'] = occurrencesAppType
        serializer.data['occurrencesAppTypeTotal'] = total

        return Response(serializer.data, status = status.HTTP_201_CREATED)

@api_view(['GET'])
def getAverageBase(request, format=None):
    obj = PredictionAverageValues.objects.filter(averageType=1).latest('creationDate')
    serializer = PredictionAverageValuesSerializer(obj)
    return Response(serializer.data)

@api_view(['GET'])
def getAverage(request, format=None):
    obj = PredictionAverageValues.objects.filter(averageType=2).latest('creationDate')
    serializer = PredictionAverageValuesSerializer(obj)
    return Response(serializer.data)

@api_view(['DELETE'])
def deleteUserAndUserInfoByUserId(request,userId, format=None):

    noInfo = False

    try:
        user = User.objects.get(userId=userId)
    except:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    try:
        userInfo = UserInfo.objects.get(userId=userId)
    except:
        userInfo = None
        noInfo = True

    if not userInfo:
        noInfo = True

    if request.method == 'DELETE':
        if noInfo == False:
            userInfo.delete()
        user.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

    return Response(status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET'])
def getStatisticsByUserId(request, userId, format=None):

    try:
        user = User.objects.get(userId=userId)
    except:
        return Response(data={"Error":"Por favor, ingrese el id del usuario"},status=status.HTTP_400_BAD_REQUEST)

    userPredictions = Prediction.objects.filter(userId=userId)

    if not userPredictions:
        return Response(data={"Error":"No se encontraron predicciones para el usuario"},status=status.HTTP_400_BAD_REQUEST)

    lastMonth = datetime.today()-timedelta(days=30)
    predNum = userPredictions.count()
    predNumThisMonth = userPredictions.filter(creationDate__gte=lastMonth).count()

    avgGpa = userPredictions.aggregate(Avg('gpaScore'))
    avgGmat = userPredictions.aggregate(Avg('gmatScore'))
    gradGpaAvg = userPredictions.aggregate(Avg('gradGpaScore'))
    workExpAvg = userPredictions.aggregate(Avg('workExp'))

    maxGpaResult = userPredictions.order_by('-gradGpaScore').first()
    maxGpaResultSerializer = PredictionSerializer(maxGpaResult)

    maxGpaResultThisMonth = userPredictions.filter(creationDate__gte=lastMonth).order_by('-gradGpaScore').first()
    maxGpaResultThisMonthSerializer = PredictionSerializer(maxGpaResultThisMonth)

    finalResponseDict = {}
    finalResponseDict['predNum'] = predNum
    finalResponseDict['predNumThisMonth'] = predNumThisMonth
    finalResponseDict['avgGpa']= round(avgGpa['gpaScore__avg'],2)
    finalResponseDict['avgGmat']=round(avgGmat['gmatScore__avg'],2)

    finalResponseDict['gradGpaAvg']=round(gradGpaAvg['gradGpaScore__avg'],2)
    finalResponseDict['workExpAvg']=round(workExpAvg['workExp__avg'],2)

    finalResponseDict['maxGpaResult']=maxGpaResultSerializer.data
    finalResponseDict['maxGpaResultThisMonth']=maxGpaResultThisMonthSerializer.data

    response = Response()
    response.data = finalResponseDict

    return response

#---------------DONT USE THESE-------------------------------------------------
@api_view(['POST'])
def insertPredictionsIntoDB(request, format=None):
    with open('D:\Culqi\Airflow\MBA-TrueSight-Backend\DatabaseLoad.csv') as f:
        reader = csv.reader(f)

        predType = PredictionType.objects.get(predictionTypeId=3)

        count = 0
        for row in reader:
            _, created = PredictionTraining.objects.get_or_create(
                predictionTrainingId=row[0],
                gmatScore=row[1],
                gpaScore=row[2],
                work_exp=row[3],
                gradGpaScore=row[4],
                OaAtGrad=row[5],
                OaAt90=row[6],
                GradYear=row[7],
                Salary=row[8],
                predictionTypeId=predType,
                appType=row[10],
                )
            count +=1
            print(count)

    response = Response()
    return (response)

@api_view(['POST'])
def predictionTesting(request,format=None):

    input_file = request.FILES['file']
    try:
        df = pd.read_csv(input_file)
    except:
         try:
            df = pd.read_excel(input_file)
         except:
             response = Response(status=status.HTTP_400_BAD_REQUEST)
             return response

    df = df.reset_index()
    headers = list(df.columns.values)
    headers = [e for e in headers if e not in ('gmat','gpa','wk_xp','app_type','index','grad_gpa')]

    finalPredictions = []
    print(headers)
    for index, row in df.iterrows():
        predictionArray = []
        singlePredictionDict ={}

        print(type(row.values))
        print(row.values)

        try:
            gmat = float(row['gmat'])
            gpa = float(row['gpa'])
            wk_xp = float(row['wk_xp'])
            app_type = float(row['app_type'])
        except:
            continue

        singlePredictionDict = {
            "gmat":gmat,
            "gpa":gpa,
            "wk_xp":wk_xp,
            "app_type":app_type,
        }

        for head in headers:
            singlePredictionDict[head] = row[head]

        print(singlePredictionDict)
    return Response()


import typing as t
import uuid
from fastapi import APIRouter, Depends, Query, status
from fermerce.app.vendor import schemas
from fermerce.app.vendor import services

from fermerce.app.staff import dependency
from fermerce.core.enum.sort_type import SearchType, SortOrder
from fermerce.core.schemas.response import ITotalCount
from fermerce.app.vendor.api import VENDOR_META

router = APIRouter(
    prefix=VENDOR_META.get("router_prefix"), tags=[VENDOR_META.get("tag")]
)

@router.get(
    "/",
    # dependencies=[Depends(dependency.require_super_admin_or_admin)],
)
async def get_users_list(
    filter_string: t.Optional[str] = Query(
        default="", alias="filter", description="filter through all attributes"
    ),
    select: t.Optional[str] = Query(
        default="", alias="select", description="select specific attributes"
    ),
    is_verified: bool = False,
    per_page: int = 10,
    page: int = 1,
    sort_by: t.Optional[SortOrder] = Query(
        default=SortOrder.desc, description="order by attribute, e.g. id"
    ),
    order_by: t.Optional[str] = Query(
        default="id", description="order by attribute, e.g. id"
    ),
    is_active: t.Optional[bool] = True,
    is_suspended: t.Optional[bool] = False,
    is_archived: t.Optional[bool] = False,
    search_type: t.Optional[SearchType] = SearchType._and,
    load_related: t.Optional[bool] = False,
):
    return await services.filter(
        filter_string=filter_string,
        per_page=per_page,
        page=page,
        select=select,
        sort_by=sort_by,
        order_by=order_by,
        is_active=is_active,
        is_suspended=is_suspended,
        is_archived=is_archived,
        search_type=search_type,
        load_related=load_related,
        is_verified=is_verified,
    )

@router.get(
    "/total/count",
    response_model=ITotalCount,
    # dependencies=[Depends(dependency.require_super_admin_or_admin)],
)
async def get_total_users() -> ITotalCount:
    return await services.get_total_Vendors()

@router.delete(
    "/",
    status_code=status.HTTP_200_OK,
    # dependencies=[Depends(dependency.require_super_admin)],
)
async def delete_vendor(data_in: schemas.IRemoveVendor) -> None:
    return await services.remove_vendor_data(data_in=data_in)

@router.get(
    "/{vendor_id}",
    status_code=status.HTTP_200_OK,
    # dependencies=[Depends(dependency.require_super_admin_or_admin)],
)
async def get_vendor(
    vendor_id: uuid.UUID,
    load_related: bool = True,
):
    return await services.get_vendor_details(vendor_id, load_related)


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.index, name="index"),
    url(r'^category/(?P<cat_id>[0-9]+)$', views.single_category, name="single_cat"),
    url(r'^(?P<books_id>[0-9]+)$', views.book, name="book"),
    url(r'^cat', views.category, name="category"),
    url(r'^login', views.login, name="login"),
]


#! /usr/bin/python2.7

# 2015.07.13
# ipython
# klassen erstellen
# stochastic gratienten descent (abstieg)
# batch GD
# use python 2.7!

import sys
import os

import numpy as np
import Tkinter as tk
import matplotlib.pyplot as plt

from random import randint
from random import uniform

from copy import deepcopy

# For colorful output in Terminal
import colorama
from colorama import Fore, Back, Style

# For clearing the Terminal
def cls():
    os.system('cls' if os.name == 'nt' else 'clear')

# Helpful for getting the max depth in a list
depth = lambda L: isinstance(L, list) and max(map(depth, L))+1

__author__ = "haris"
__date__ = "$Jun 24, 2015 11:10:57 AM$"

## Class for Network
## Class for Weights
## Class for Biases

class NeuralNetwork(Exception):

    error_not_init = "Please, initial first the matrices!\nUse e.g. init_random() or other functions"

    def scalar_matrix_multiplication(self, scalar, matrix):
        scalar_matrix = [[scalar for i2 in range(0, len(list(matrix[0])))] for i1 in range(0, len(list(matrix)))]
        scalar_matrix = np.array(scalar_matrix)

        return scalar_matrix * matrix

    def f(self, a) : return (1 / (1 + np.exp(-a)))

    def fi(self, a) : return (self.f(a) * self.f(1 - a))

    def sigmoid(self, vector):
        new_vector_list = [val.tolist()[0] for val in vector]
        return np.array([[self.f(v)] for v in new_vector_list])

    def sigmoid_invert(self, vector):
        new_vector_list = [val.tolist()[0] for val in vector]
        return np.array([[self.fi(v)] for v in new_vector_list])

    def square_error(self, v1, v2):
        square_error = 0
        for n1, n2 in zip(v1.transpose().tolist()[0], v2.transpose().tolist()[0]):
            square_error += (float(n1) - float(n2))**2
        return square_error

    # # Sigmoid Function
    # def f(self, a):
    #     return 1 / (1 + np.e**(-a))
    # # Sigmoid Derivative
    # def fi(self, a):
    #     return self.f(a) * self.f(1-a)

    def __init__(self, inputs = 1, outputs = 1):
        self.inputs = inputs
        self.outputs = outputs
        self.hidden = []
        self.matrix_init = False
        self.weights_init = False
        self.targets_init = False
        self.neuron_list = [inputs, outputs]
        self.weights = []
        self.biases = []
        self.targets_amount = 0
        self.io_targets = []
        self.learning_rate = 0.005
        self.show_plot = True

    def stats(self):
        print("inputs = " + str(self.inputs))
        print("outputs = " + str(self.outputs))
        if len(self.hidden) == 0:
            print("No hidden layer")
        else:
            print(str(len(self.hidden)) + " layers set with " + str(self.hidden) + " neurons")
        print("Neuronlist: "+str(self.neuron_list))
        if self.matrix_init == True:
            print("Matrix is initialized:")
            print("Biases and Weights:")
            for i in xrange(0, len(self.weights)):
                print("For Layer "+str(i+1)+" with Format ("+str(self.neuron_list[i])+","+str(self.neuron_list[i+1])+")")
                print("Biase:\n"+str(self.biases[i]))
                print("Weight:\n"+str(self.weights[i]))
        else:
            print("Matrix is not initialized!!!")

        print("Plot is set to "+str(self.show_plot))

    def __get_random_matrix(self, rows, columns, min, max, roundCom):
        matrix = []

        i = 0
        while i < rows:
            tmp = []
            j = 0
            while j < columns:
                tmp.append(round(uniform(min, max), roundCom))
                j += 1
            matrix.append(tmp)
            i += 1

        return matrix

    def get_random_matrix(self, rows, columns, min, max, roundCom):
        matrix = []

        i = 0
        while i < rows:
            tmp = []
            j = 0
            while j < columns:
                tmp.append(round(uniform(min, max), roundCom))
                j += 1
            matrix.append(tmp)
            i += 1

        return matrix

    # Getter
    def get_input_layer(self):
        return self.inputs

    def get_output_layer(self):
        return self.outputs

    def get_neuron_list(self):
        return self.neuron_list

    def get_random_biases_list(self, neuron_list, nmin = -3, nmax = 3, ncommas = 3):
        nl = neuron_list
        biases = [np.array(self.__get_random_matrix(1, nl[i + 1], nmin, nmax, ncommas)) for i in xrange(0, len(nl) - 1)]
        return biases

    def get_random_weight_list(self, neuron_list, nmin = -3, nmax = 3, ncommas = 3):
        nl = neuron_list
        weights = [np.array(self.__get_random_matrix(nl[i], nl[i + 1], nmin, nmax, ncommas)) for i in xrange(0, len(nl) - 1)]
        return weights

    def get_random_neurons_weights(self, neuron_list, nmin = -3, nmax = 3, ncommas = 3):
        nl = neuron_list
        biases = [np.array(self.__get_random_matrix(1, nl[i + 1], nmin, nmax, ncommas)) for i in xrange(0, len(nl) - 1)]
        weights = [np.array(self.__get_random_matrix(nl[i], nl[i + 1], nmin, nmax, ncommas)) for i in xrange(0, len(nl) - 1)]
        return biases, weights

    def get_network_error(self, network, inputs, targets):
        nl, bs, ws = network[0], network[1], network[2]

        outl = [self.calculate_forward(i, bs, ws) for i in inputs]

        errl = [self.square_error(out, t) for out, t in zip(outl, targets)]

        error_total = 0
        for e in errl:
            error_total += e

        return error_total

    # Setter
    def set_neuron_list(self, inputs):# for input, hidden and output layer simultaneously
        self.matrix_init = False
        self.set_input_layer(inputs[0])
        self.set_hidden_layer(inputs[1:-1])
        self.set_output_layer(inputs[-1])
        self.neuron_list = inputs

    def set_input_layer(self, inputs):
        self.matrix_init = False
        self.inputs = inputs

    def set_output_layer(self, outputs):
        self.matrix_init = False
        self.outputs = outputs

    def set_hidden_layer(self, numbers):
        if type(numbers) != type([]):
            print("Must be type of lists! Hidden layer was not changed!")
        if len(numbers) < 0:
            print("No negative number possible!")
        elif len(numbers) > 10:
            print("No more than 10 Layer possible! Set to 10 hidden Layers")
            self.hidden = numbers[0:10]
        else:
            print("Set amount of hidden layer to " + str(len(numbers)))
            self.hidden = numbers

    def init_random_weights(self):
        # Create a network with hidden layer
        nl = self.neuron_list
        self.weights = []
        self.biases = []
        for i in range(0, len(nl) - 1):
            self.weights.append(np.array(self.__get_random_matrix(nl[i], nl[i + 1], -3, 3, 1)))
            self.biases.append(np.array(self.__get_random_matrix(1, nl[i + 1], -3, 3, 1)))

        self.weights_init = True
        print("Created random biases and weights for "+str(self.neuron_list)+" neural list")

    def init_random_targets(self):
        self.targets_amount = randint(1, 100)
        nl = self.neuron_list
        ta = self.targets_amount
        inputs = [np.array(self.__get_random_matrix(nl[0], 1, -3, 3, 3))
                  for _ in range(0, ta)]
        targets = [np.array(self.__get_random_matrix(nl[-1], 1, 0.1, 0.9, 3))
                   for _ in range(0, ta)]

        self.targets_init = True
        print("Create "+str(self.targets_amount+" inputs and targets"))

    # Calculate a given network through
    # param weights: Weights of the neurons with bias
    # param input: Input of the first Layer of the Network
    def calculate_forward(self, ninput, biases, weights):
        x = ninput
        for i in range(0, len(weights)):
            b = biases[i]
            w = weights[i]
            # b = np.outer(biases[i], np.ones(len(w[0])))
            wi = np.array(list(b) + list(w))
            x = np.array([[1]] + list(x))
            x = np.dot(wi.transpose(), x)
            x = self.sigmoid(x)
        return x

    def backpropagation(self, start_input, biases, weights, targets, etha):
        # Feed forward
        xs = []
        ys = [start_input]

        y = start_input
        for i in xrange(0, len(weights)):
            x = y
            b = biases[i]
            w = weights[i]
            # b = np.outer(biases[i], np.ones(len(w[0])))
            wi = np.array(list(b) + list(w))
            x = np.array([[1]] + list(x))
            a = np.dot(wi.transpose(), x)
            xs.append(a)
            y = self.sigmoid(a)
            ys.append(y)

        # Backward error correction
        bs = deepcopy(biases)
        ws = deepcopy(weights)

        d = np.subtract(ys[-1], targets) * self.sigmoid_invert(xs[-1])
        bs[-1] = d.transpose()

        ws[-1] = np.dot(d, ys[-2].transpose())

        for i in xrange(2, len(weights) + 1):
            x = xs[-i]
            xp = self.sigmoid_invert(x)

            d = np.dot(weights[-i + 1], d) * xp

            bs[-i] = d.transpose()
            ws[-i] = np.dot(d, ys[-i-1].transpose()).transpose()

        bs = [self.scalar_matrix_multiplication(etha, bs[i]) for i in xrange(0, len(biases))]

        ws[-1] = ws[-1].transpose()

        ws = [self.scalar_matrix_multiplication(etha, ws[i]) for i in xrange(0, len(weights))]
        return (bs, ws)

    def calculate_with_initial_values_sgd(self):
        if self.matrix_init is False:
            print(self.error_not_init)

    def calculate_with_initial_values_bgd(self):
        if self.matrix_init is False:
            print(self.error_not_init)

    # Random Part, is for testing the Neural Network
    def improve_network(self, network, inputs, targets, etha, iterations, show_plot = False, show_debug_msg = False):
        nl, bs, ws = network[0], network[1], network[2]

        testing_amount = len(inputs)

        if show_debug_msg is True:
            print("Creating "+str(testing_amount)+" inputs and targets...")

        outputs = [self.calculate_forward(inputs[j], bs, ws) for j in range(0, testing_amount)]

        errors_temp = [self.square_error(outputs[j], targets[j]) for j in range(0, testing_amount)]
        error_total = 0
        for j in range(0, testing_amount):
            error_total += errors_temp[j]

        divide_factor = testing_amount * nl[-1]
        error_total /= divide_factor
        errors = [error_total]
        diffs = []
        diffs2 = []
        last_error = error_total
        last2_error = error_total
        bsd = [0 for j in range(0, testing_amount)]
        wsd = [0 for j in range(0, testing_amount)]

        for i in range(0, iterations):
            if i % 1000 == 0 and show_debug_msg is True:
                print("i = "+str(i)+"     error_total = "+str(error_total)+"    error_diff = "+str(last_error-error_total)+"   error diff2 = "+str((last2_error-last_error)-(last_error-error_total)))

            for j in range(0, testing_amount):
                bsd[j], wsd[j] = self.backpropagation(inputs[j], bs, ws, targets[j], etha)

            for j in range(0, testing_amount):
                for k in range(0, len(nl) - 1):
                    bs[k] = np.subtract(bs[k], bsd[j][k])
                    ws[k] = np.subtract(ws[k], wsd[j][k])

            outputs = [self.calculate_forward(inputs[j], bs, ws) for j in range(0, testing_amount)]
            errors_temp = [self.square_error(outputs[j], targets[j]) for j in range(0, testing_amount)]

            last2_error = last_error
            last_error = error_total
            error_total = 0
            for j in range(0, testing_amount):
                error_total += errors_temp[j]
            error_total /= divide_factor
            errors.append(error_total)
            diffs.append(last_error-error_total)
            diffs2.append((last2_error-last_error)-(last_error-error_total))
        # print("last error_total in improve_network is "+str(errors[-1]))

        if self.show_plot is True and show_plot is True:
            p_x = [i for i in range(0, len(errors))]
            p_y = [float(e) for e in errors]
            p_x_dif = [i for i in range(0, len(diffs))]
            p_y_dif = [d for d in diffs]
            p_x_dif2 = [i for i in range(0, len(diffs2))]
            p_y_dif2 = [d2 for d2 in diffs2]

            plt.figure(0)

            ax1 = plt.subplot(111)
            ax1.plot(p_x, p_y)
            plt.yscale('log')
            plt.title('Learning curve of the Neural Network')
            plt.ylabel('Absolute Error from inputs\nto current outputs and targets')
            plt.xlabel('Number of Iterations')
            # plt.figure(1)

            plt.figure(1)
            ax2 = plt.subplot(211)
            ax2.plot(p_x_dif, p_y_dif)
            plt.yscale('log')
            # ax2.yscale('log')

            # plt.figure(2)
            ax3 = plt.subplot(212)
            ax3.plot(p_x_dif2, p_y_dif2)
            plt.yscale('log')
            # ax3.yscale('log')
            plt.show()

        return nl, bs, ws

    def iterate_with_sgd(self, neuron_list, etha, testing_amount, iterations):
        self.__iterate_with_random_generated_values(neuron_list, etha, testing_amount, iterations, True)

    def iterate_with_bgd(self, neuron_list, etha, testing_amount, iterations):
        self.__iterate_with_random_generated_values(neuron_list, etha, testing_amount, iterations, False)

    def __iterate_with_random_generated_values(self, neuron_list, etha, testing_amount, iterations, is_sgd):
        nl = neuron_list

        print("Creating "+str(testing_amount)+" inputs and targets...")

        inputs = [np.array(self.__get_random_matrix(nl[0],1,-3,3,3)) for i in range(0, testing_amount)]
        if is_sgd is True:
            bref, wref = self.get_random_neurons_weights(nl)
            targets = [self.calculate_forward(inputs[i], bref, wref) for i in range(0, testing_amount)]
        else:
            targets = [np.array(self.__get_random_matrix(nl[-1],1,0.1,0.9,3)) for i in range(0, testing_amount)]

        bs, ws = self.get_random_neurons_weights(nl)
        outputs = [self.calculate_forward(inputs[j], bs, ws) for j in range(0, testing_amount)]

        errors_temp = [self.square_error(outputs[j], targets[j]) for j in range(0, testing_amount)]
        error_total = 0
        for j in range(0, testing_amount):
            error_total += errors_temp[j]

        divide_factor = testing_amount * neuron_list[-1]
        error_total /= divide_factor
        errors = [error_total]
        diffs = []
        diffs2 = []
        last_error = error_total
        last2_error = error_total
        bsd = [0 for j in range(0, testing_amount)]
        wsd = [0 for j in range(0, testing_amount)]

        for i in range(0, iterations):
            if i % 1000 == 0:
                print("i = "+str(i)+"     error_total = "+str(error_total)+"    error_diff = "+str(last_error-error_total)+"   error diff2 = "+str((last2_error-last_error)-(last_error-error_total)))

            if is_sgd is True:
                inputs = [np.array(self.__get_random_matrix(nl[0],1,-3,3,3)) for j in range(0, testing_amount)]
                targets = [self.calculate_forward(inputs[j], bref, wref) for j in range(0, testing_amount)]

            for j in range(0, testing_amount):
                bsd[j], wsd[j] = self.backpropagation(inputs[j], bs, ws, targets[j], etha)

            for j in range(0, testing_amount):
                for k in range(0, len(nl) - 1):
                    bs[k] = np.subtract(bs[k], bsd[j][k])
                    ws[k] = np.subtract(ws[k], wsd[j][k])

            outputs = [self.calculate_forward(inputs[j], bs, ws) for j in range(0, testing_amount)]
            errors_temp = [self.square_error(outputs[j], targets[j]) for j in range(0, testing_amount)]

            last2_error = last_error
            last_error = error_total
            error_total = 0
            for j in range(0, testing_amount):
                error_total += errors_temp[j]
            error_total /= divide_factor
            errors.append(error_total)
            diffs.append(last_error-error_total)
            diffs2.append((last2_error-last_error)-(last_error-error_total))
        p_x = [i for i in range(0, len(errors))]
        p_y = [float(e) for e in errors]
        p_x_dif = [i for i in range(0, len(diffs))]
        p_y_dif = [d for d in diffs]
        p_x_dif2 = [i for i in range(0, len(diffs2))]
        p_y_dif2 = [d2 for d2 in diffs2]

        if self.show_plot is True:
            plt.figure(0)

            ax1 = plt.subplot(111)
            ax1.plot(p_x, p_y)
            plt.yscale('log')
            plt.title('Learning curve of the Neural Network')
            plt.ylabel('Absolute Error from inputs\nto current outputs and targets')
            plt.xlabel('Number of Iterations')
            # plt.figure(1)

            plt.figure(1)
            ax2 = plt.subplot(211)
            ax2.plot(p_x_dif, p_y_dif)
            plt.yscale('log')
            # ax2.yscale('log')

            # plt.figure(2)
            ax3 = plt.subplot(212)
            ax3.plot(p_x_dif2, p_y_dif2)
            plt.yscale('log')
            # ax3.yscale('log')
            plt.show()

# ## Testing som functions
# # Create a network Object
# nn = NeuralNetwork()

# # Set some varibales
# nl = [5,4,5]
# bs = nn.get_random_biases_list(nl)
# ws = nn.get_random_weight_list(nl)
# ### Is also possible
# ### bs, ws = nn.get_random_neuron_weight(nl)

# # Get the whole Network
# network = [nl, bs, ws]

# # Set the amount of testing Points (Values)
# testing_amount = 10

# # Get the Inputs and Targets values for learing the Network (here are the values random)
# inputs = [np.array(nn.get_random_matrix(nl[0],1,-3,3,3)) for i in range(0, testing_amount)]
# targets = [np.array(nn.get_random_matrix(nl[-1],1,0.1,0.9,3)) for i in range(0, testing_amount)]

# # Test a loop for learing (is hopfully working ^^)
# print("testing my network with the function improve_network")
# for i in range(0, 4):
#     network = nn.improve_network(network, inputs, targets, 0.05, 1000)
#     print("error of the network is: "+str(nn.get_network_error(network, inputs, targets) / testing_amount / nl[-1]))
# # nn.iterate_with_bgd([5,5,5], 0.05, 10, 1000)


from datetime import datetime, timedelta
import unittest

if __name__ == "__main__":
    import utils
    utils.import_depends()

from brokertest import TestBrokerCommand

EXPIRY = datetime.utcnow().replace(microsecond=0) + timedelta(days=1)
EXPIRY = EXPIRY.isoformat().replace("T", " ")

class TestAddRebootIntervention(TestBrokerCommand):

    def test_00_basic_reboot_intervention(self):
        command = ["show_reboot_intervention",
                   "--hostname=server1.aqd-unittest.ms.com"]
        out = self.notfoundtest(command)

        command = ["add_reboot_intervention", "--expiry", EXPIRY,
                   "--reason=test",
                   "--hostname=server1.aqd-unittest.ms.com"]
        self.successtest(command)

        command = ["show_reboot_intervention",
                   "--hostname=server1.aqd-unittest.ms.com"]
        out = self.commandtest(command)
        self.searchoutput(out, "RebootIntervention$", command)
        self.matchoutput(out, "Bound to: Host server1.aqd-unittest.ms.com",
                         command)
        self.matchoutput(out, "Start: ", command)
        self.matchoutput(out, "Expiry: ", command)

        command = ["cat", "--reboot_intervention",
                   "--hostname=server1.aqd-unittest.ms.com"]
        out = self.commandtest(command)
        self.matchoutput(out,
                         "structure template resource"
                         "/host/server1.aqd-unittest.ms.com"
                         "/reboot_iv/reboot_intervention/config;",
                         command)
        self.matchoutput(out, "\"name\" = \"reboot_intervention\";", command)
        self.matchoutput(out, "\"start\" =", command)
        self.matchoutput(out, "\"expiry\" =", command)

        command = ["cat", "--reboot_intervention",
                   "--hostname=server1.aqd-unittest.ms.com",
                   "--generate"]
        newout = self.commandtest(command)
        self.assertEqual(out, newout)

        command = ["show_reboot_intervention", "--all"]
        out = self.commandtest(command)
        self.searchoutput(out, "RebootIntervention$", command)

    def test_11_addexisting(self):
        EXPIRY = datetime.utcnow().replace(microsecond=0) + timedelta(days=2)
        command = ["add_reboot_intervention", "--expiry", EXPIRY,
                   "--reason=test",
                   "--hostname=server1.aqd-unittest.ms.com"]
        out = self.badrequesttest(command)
        self.matchoutput(out, "already exists", command)

    def test_12_addbadtime(self):
        command = ["add_reboot_intervention", "--start_time=2013/01/01",
                   "--expiry=2013/01/14",
                   "--reason=test",
                   "--hostname=server2.aqd-unittest.ms.com"]
        out = self.badrequesttest(command)
        self.matchoutput(out, "The start time or expiry time are in the past.", command)

    def test_15_notfoundri(self):
        command = ["cat", "--reboot_intervention",
                   "--hostname=server3.aqd-unittest.ms.com"]
        self.notfoundtest(command)

    def test_30_checkthehost(self):
        command = ["show_host", "--host=server1.aqd-unittest.ms.com"]
        out = self.commandtest(command)
        self.matchoutput(out, "RebootIntervention", command)

        command = ["make", "--hostname=server1.aqd-unittest.ms.com"]
        out = self.successtest(command)

        command = ["cat", "--generate",
                   "--hostname", "server1.aqd-unittest.ms.com", "--data"]
        out = self.commandtest(command)
        self.matchoutput(out,
                         '"system/resources/reboot_iv" = '
                         'append(create("resource/host/server1.aqd-unittest.ms.com/reboot_iv/reboot_intervention/config"))',
                         command)

        command = ["show_host", "--hostname=server1.aqd-unittest.ms.com",
                   "--format=proto"]
        host = self.protobuftest(command, expect=1)[0]
        found = False
        for resource in host.resources:
            if resource.name == "reboot_intervention" and \
               resource.type == "reboot_iv":
                found = True
        self.assertTrue(found, "No reboot_iv found in host protobuf.")

if __name__ == '__main__':
    suite = unittest.TestLoader().loadTestsFromTestCase(
        TestAddRebootIntervention)
    unittest.TextTestRunner(verbosity=2).run(suite)