from keras.models import Model
from keras.layers.recurrent import LSTM
from keras.layers import Dense, Input, Embedding
from keras.preprocessing.sequence import pad_sequences
from keras.callbacks import ModelCheckpoint
from collections import Counter
import nltk
import numpy as np
from sklearn.model_selection import train_test_split
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import os
import glob

np.random.seed(42)

## Change the directory addresses, hidden units and test dataset ####
HIDDEN_UNITS = 50
TESTING_DATASET = 'dlgs_trial5.txt'
WEIGHTS_DIRECTORY_NAME = 'dlgs_trial5'

CWD = os.getcwd()
FINAL_RESULTS_FILE = CWD+'/'+WEIGHTS_DIRECTORY_NAME+'_all_weights.txt'

c= np.load('word-context.npy')
context = c.item()

num_encoder_tokens = context['num_encoder_tokens']
num_decoder_tokens = context['num_decoder_tokens']
encoder_max_seq_length = context['encoder_max_seq_length']
decoder_max_seq_length = context['decoder_max_seq_length']

input_word2idx1 = np.load('input-word2idx.npy')
input_word2idx = input_word2idx1.item()
input_idx2word1 = np.load('input-idx2word.npy')
input_idx2word = input_idx2word1.item()
target_word2idx1 = np.load('target-word2idx.npy')
target_word2idx = target_word2idx1.item()
target_idx2word1 = np.load('target-idx2word.npy')
target_idx2word = target_idx2word1.item()

##========NETWORK ARCHITECTURE===================================

encoder_inputs = Input(shape=(None,), name='encoder_inputs')
encoder_embedding = Embedding(input_dim=num_encoder_tokens, output_dim=HIDDEN_UNITS,
                              input_length=encoder_max_seq_length, name='encoder_embedding')
encoder_lstm = LSTM(units=HIDDEN_UNITS, return_state=True, name='encoder_lstm')
encoder_outputs, encoder_state_h, encoder_state_c = encoder_lstm(encoder_embedding(encoder_inputs))
encoder_states = [encoder_state_h, encoder_state_c]

decoder_inputs = Input(shape=(None, num_decoder_tokens), name='decoder_inputs')
decoder_lstm = LSTM(units=HIDDEN_UNITS, return_state=True, return_sequences=True, name='decoder_lstm')
decoder_outputs, decoder_state_h, decoder_state_c = decoder_lstm(decoder_inputs,
                                                                 initial_state=encoder_states)
decoder_dense = Dense(units=num_decoder_tokens, activation='softmax', name='decoder_dense')
decoder_outputs = decoder_dense(decoder_outputs)

model = Model([encoder_inputs, decoder_inputs], decoder_outputs)

model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])

## =============== DECODER FUNCTION ===========================
def decode_sequence(input_text):
    input_seq = []
    input_wids = []
    for word in nltk.word_tokenize(input_text):
        idx = 1  # default [UNK]
        if word in input_word2idx:
            idx = input_word2idx[word]
        input_wids.append(idx)
    input_seq.append(input_wids)
    input_seq = pad_sequences(input_seq, encoder_max_seq_length)

    states_value = encoder_model.predict(input_seq)

    target_seq = np.zeros((1, 1, num_decoder_tokens))
    target_seq[0, 0, target_word2idx['START']] = 1

    target_text = ''
    target_text_len = 0
    terminated = False
    while not terminated:
        output_tokens, h, c = decoder_model.predict([target_seq] + states_value)

        sample_token_idx = np.argmax(output_tokens[0, -1, :])

        sample_word = target_idx2word[sample_token_idx]

        target_text_len += 1

        if sample_word != 'START' and sample_word != 'END':
            target_text += ' ' + sample_word

        if sample_word == 'END' or target_text_len >= decoder_max_seq_length:
            terminated = True

        target_seq = np.zeros((1, 1, num_decoder_tokens))
        target_seq[0, 0, sample_token_idx] = 1

        states_value = [h, c]

    return target_text.strip()

##========== Load Weight File ========= ####
os.chdir(WEIGHTS_DIRECTORY_NAME)

if os.path.isfile(FINAL_RESULTS_FILE):
    os.remove(FINAL_RESULTS_FILE)

output_file = open(FINAL_RESULTS_FILE,'a')
weight_files = glob.glob('*.h5')

for weight_file in weight_files:
    model.load_weights(weight_file)
    output_file.write("\n==========================================")
    output_file.write("\n\t"+weight_file)
    output_file.write("\n==========================================")

    encoder_model = Model(encoder_inputs, encoder_states)
    decoder_state_inputs = [Input(shape=(HIDDEN_UNITS,)), Input(shape=(HIDDEN_UNITS,))]
    decoder_outputs, state_h, state_c = decoder_lstm(decoder_inputs, initial_state=decoder_state_inputs)
    decoder_states = [state_h, state_c]
    decoder_outputs = decoder_dense(decoder_outputs)
    decoder_model = Model([decoder_inputs] + decoder_state_inputs, [decoder_outputs] + decoder_states)

    with open(TESTING_DATASET, 'r', encoding='utf-8') as f:
        lines1 = f.read().split('\n')

    length_of_file= sum(1 for line in open(TESTING_DATASET))
    for line in lines1[: min(length_of_file,len(lines1) - 1)]:
        input_text, target_text = line.split('\t')
        decoded_sentence = decode_sequence(input_text)
        user_input = 'Input sentence:'+input_text
        bot_output = 'Decoded sentence:'+decoded_sentence
        # print('-')
        # print(user_input)
        # print(bot_output)
        output_file.write("\n-")
        output_file.write('\n'+user_input)
        output_file.write('\n'+bot_output)

    print('Tested with weights in : ', weight_file)

output_file.close()


from discord.ext import commands
import discord
import random

class HelpCog(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.command(aliases=['хелп'])
    async def help(self, ctx):
        prefix = '!'
        emb = discord.Embed(title='Команды сервера: ', description=f'`{prefix}help`', color=discord.Color(random.randint(0x000000, 0xFFFFFF)), inline=False)
        emb.set_thumbnail(url=self.bot.user.avatar_url)
        emb.add_field(name='Модерация: только для роли [Админ] ',
                      value=f'`{prefix}clear` `{prefix}add_role` `{prefix}ban` `{prefix}mute` `{prefix}unmute` `{prefix}call`',
                      inline=False)
        emb.add_field(name='Развлечения: ',
                      value=f'`{prefix}cat` `{prefix}quote`',
                      inline=False)
        emb.add_field(name='Прочее: ',
                      value=f'`{prefix}info` `{prefix}my_info`',
                      inline=False)
        await ctx.send(embed=emb)

def setup(bot):
    bot.add_cog(HelpCog(bot))


import os
import sys

import inference
import storage
import boto3

# Flask
from flask import Flask, redirect, url_for, request, render_template, Response, jsonify, redirect
from werkzeug.utils import secure_filename
from gevent.pywsgi import WSGIServer

# Some utilites
import numpy as np
from util import base64_to_pil

TCP_PORT = 5000
aws_access_key_id=''
aws_secret_access_key=''
# Declare a flask app
app = Flask(__name__)

s3 = boto3.client('s3',aws_access_key_id = aws_access_key_id, aws_secret_access_key = aws_secret_access_key)
inference_handler = inference.Inference(s3)
print('Model loaded. Check http://127.0.0.1:'+str(TCP_PORT))

@app.route('/', methods=['GET'])
def index():
    # Main page
    return render_template('index.html')

@app.route('/predict', methods=['GET', 'POST'])
def predict():
    if request.method == 'POST':
        img = base64_to_pil(request.json)
        hash_value = str(hex(hash(img.tobytes())))
        response = inference_handler.predict(img)
        storage.temp_store(hash_value, img)

        return jsonify(result=response, hash_value=hash_value)

    return None

@app.route('/feedback', methods=['GET', 'POST'])
def feedback():
    global s3
    if request.method == 'POST':
        hash_value = request.json['hash_value']
        correct_label = str.lower(request.json['label'])
        if(correct_label in inference_handler.classes):
            storage.copy_file(correct_label, hash_value)
        storage.remove_file(hash_value)
        resp = jsonify(success=True)
        return resp

    return None

if __name__ == '__main__':

    http_server = WSGIServer(('0.0.0.0', TCP_PORT), app)
    http_server.serve_forever()


from flask import make_response
from google.cloud import bigquery

# my modules
import config
from my_logging import getLogger

log = getLogger(__name__)
config = config.Config()

with open('auth.txt', 'r') as f:
    cf_token = f.readline().rstrip('\r\n')

# entry point of Cloud Functions
# trigger = http
def atm_iv_data(request):
    if not check_auth(request):
        return 'Forbidden', 403

    num_days = request.args.get('d', default='7')
    n_th_contract_month = request.args.get('n', default='0')

    df = query_atm_iv(num_days, n_th_contract_month)

    # CSV化
    atm_iv_csv = df.to_csv(index=False, header=False)

    res = make_response(atm_iv_csv, 200)
    res.headers['Content-type'] = 'text/csv; charset=utf-8'
    return res

def query_atm_iv(num_days, n_th_contract_month=0):
    table_iv = f'{config.gcp_bq_dataset_name}.atm_iv'
    table_option_price = f'{config.gcp_bq_dataset_name}.option_price'

    query = (f'''
        WITH t1 AS (
            SELECT
                (ARRAY_AGG(DISTINCT(last_trading_day) IGNORE NULLS ORDER BY last_trading_day ASC))[OFFSET({n_th_contract_month})] AS last_trading_day
            FROM
                {table_iv}
            WHERE
                started_at > TIMESTAMP_TRUNC(TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL -10 DAY), DAY)
                AND last_trading_day >=  CURRENT_DATE('Asia/Tokyo')
        ), t2 AS (
            SELECT
                open,high,low,close, started_at
            FROM
                {table_iv}
            WHERE
                last_trading_day=(SELECT t1.last_trading_day FROM t1)
                AND started_at >= TIMESTAMP_TRUNC(TIMESTAMP_ADD(CURRENT_TIMESTAMP() , INTERVAL -{num_days} DAY), DAY)
        ), t3 AS (
            -- まだ時間足集計が保存されていない分(=UTCでの当日分)の時間足を集計する
            SELECT
                (array_agg(iv IGNORE NULLS ORDER BY created_at ASC))[OFFSET(0)] open,
                MAX(iv) high,
                MIN(iv) low,
                (array_agg(iv IGNORE NULLS ORDER BY created_at DESC))[OFFSET(0)] close,
                TIMESTAMP_SECONDS(CAST(TRUNC(UNIX_SECONDS(created_at)/3600) AS INT64) * 3600) AS started_at
            FROM
                {table_option_price}
            WHERE
                created_at >= TIMESTAMP_TRUNC(CURRENT_TIMESTAMP(), DAY)
                AND is_atm=True AND type=2 AND last_trading_day=(SELECT t1.last_trading_day FROM t1)
            GROUP BY started_at
        )
        SELECT
            open, high, low, close, started_at
        FROM
            t2

        UNION ALL

        SELECT
            open, high, low, close, started_at
        FROM
            t3
        ORDER BY started_at
    ''')

    client = bigquery.Client()
    query_job = client.query(query)
    rows = query_job.result()
    df = rows.to_dataframe()

    return df

def check_auth(request):
    auth_header = request.headers.get("Authorization")

    req_cf_token = None

    if auth_header is not None:
        req_cf_token = auth_header.split(' ')[1]

    return req_cf_token == cf_token

if __name__ == '__main__':
    from flask import Flask, request

    app = Flask(__name__)

    @app.route('/')
    def index():
        return atm_iv_data(request)

    app.run('127.0.0.1', 8000, debug=True)


import tensorflow as tf
from pathlib import Path
import numpy as np
import dataset
import tensorflow_probability as tfp

tfd = tfp.distributions

class Encoder:

    def __init__(self, latent_size):
        super(Encoder, self).__init__()

        # static parameters
        self.latent_size = latent_size
        self.num_channels = 1
        self.dimensionality = 3
        self.fmap_base = 4096
        self.fmap_max = 512

        # dynamic parameters
        self.current_resolution = 1
        self.current_width = 2 ** self.current_resolution
        self.growing_encoder = self.make_Ebase(nf=self._nf(1))
        self.train_encoder = tf.keras.Sequential(self.growing_encoder,name='sequential')

    def update_res(self):
        self.current_resolution += 1
        self.current_width = 2 ** self.current_resolution

    def make_Ebase(self,nf):

        images = tf.keras.layers.Input(shape= (2,)*self.dimensionality + (nf,), name='images_2iso')

        # Final dense layer
        x = tf.keras.layers.Flatten()(images)
        x = tf.keras.layers.Dense(2*self.latent_size,activation=None)(x)
        return tf.keras.models.Model(inputs=[images], outputs=[x], name='z_params')

    def make_Eblock(self,name,nf):

        # on fait cette approche car on ne sait pas la taille donc on met pas un input
        block_layers = []

        #block_layers.append(tf.keras.layers.Convolution3D(nf, kernel_size=3, strides=1, padding='same'))
        #block_layers.append(tf.keras.layers.Activation(tf.nn.leaky_relu))

        block_layers.append(tf.keras.layers.Convolution3D(nf, kernel_size=4, strides=2, padding='same')) # check padding
        block_layers.append(tf.keras.layers.Activation(tf.nn.leaky_relu))

        return tf.keras.models.Sequential(block_layers, name=name)

    def _nf(self, stage):
        # computes number of filters for each layer
        return min(int(self.fmap_base / (2.0 ** (stage))), self.fmap_max)

    def _weighted_sum(self):
        return tf.keras.layers.Lambda(lambda inputs : (1-inputs[2])*inputs[0] + (inputs[2])*inputs[1])

    def add_resolution(self):

        # Add resolution
        self.update_res()

        # Gan images
        images = tf.keras.layers.Input(shape=(self.current_width,)*self.dimensionality+ (self.num_channels,),name = 'GAN_images')
        alpha = tf.keras.layers.Input(shape=[], name='e_alpha')

        # Compression block
        name = 'block_{}'.format(self.current_resolution)
        e_block = self.make_Eblock(name=name,nf=self._nf(self.current_resolution-1))

        # Channel compression
        from_rgb_1 = tf.keras.layers.AveragePooling3D()(images)
        from_rgb_1 = tf.keras.layers.Conv3D(self._nf(self.current_resolution-1), kernel_size=1, padding='same', name='from_rgb_1')(from_rgb_1)

        from_rgb_2 = tf.keras.layers.Conv3D(self._nf(self.current_resolution), kernel_size=1, padding='same', name='from_rgb_2')(images)
        from_rgb_2 = e_block(from_rgb_2)

        lerp_input = self._weighted_sum()([from_rgb_1, from_rgb_2, alpha]) # RANDOM ALPHA

        # Getting latent code
        e_z = self.growing_encoder(lerp_input)

        # Updating the model
        self.growing_encoder = tf.keras.Sequential([e_block,self.growing_encoder]) # without channel compression
        self.train_encoder = tf.keras.Model(inputs=[images,alpha],outputs=[e_z]) # with channel compression

class Decoder():

    def __init__(self, latent_size, generator_folder):
        super(Decoder, self).__init__()

        # static parameters
        self.latent_size = latent_size
        self.model_folder = generator_folder

        # dynamic parameters
        self.current_resolution = 1
        self.current_width = 2** self.current_resolution
        self.decoder = None

    def get_model(self,folder,res):
        # find the model for the appropriate resolution
        path = Path(folder)
        return str(list(path.glob('g_{}.h5'.format(res)))[0].resolve()) # check if single one

    def update_res(self):
        self.current_resolution += 1
        self.current_width = 2 ** self.current_resolution

    def make_Dblock(self,name):

        block_layers = []
        block_layers.append(tf.keras.layers.Flatten()) # obligé ??
        block_layers.append(tf.keras.layers.Dense(self.current_width**3))
        block_layers.append(tf.keras.layers.Reshape((self.current_width,self.current_width,self.current_width),input_shape=(self.current_width**3,)))
        #block_layers.append(tf.keras.layers.Activation(tf.nn.leaky_relu)) - depends on expression of NLL loss

        return tf.keras.models.Sequential(block_layers, name=name)

    def add_resolution(self):
        self.update_res()

        if self.current_resolution > 2:

            latent = tf.keras.layers.Input(shape=self.latent_size)
            alpha = tf.keras.layers.Input(shape=[], name='d_alpha')
            common = tf.keras.models.load_model(self.get_model(self.model_folder,self.current_resolution), custom_objects={'leaky_relu': tf.nn.leaky_relu}, compile=True)([latent,alpha])
            shape = tf.shape(common)
            common = tf.reshape(common,[shape[0],common.shape[1],common.shape[2],common.shape[3]])
            mu = self.make_Dblock(name='mu_block')(common)
            sigma = self.make_Dblock(name='sigma_block')(common)

            self.decoder = tf.keras.Model(inputs=[latent,alpha],outputs=[mu,sigma])
            self.decoder.trainable = True

    def get_decoder(self):
        return self.decoder

    def get_currentres(self):
        return self.current_resolution

class Generator():

    def __init__(self, latent_size, generator_folder):
        super(Generator,self).__init__()

        #static parameters
        self.latent_size = latent_size  # LOOK UP WHAT SIZE
        self.model_folder = generator_folder

        # dynamic
        self.current_resolution = 1
        self.current_width = 2**3
        self.generator = None

    def get_model(self,folder,res):
        # find the model for the appropriate resolution
        path = Path(folder)
        try:
            return str(list(path.glob('**/g_{}.h5'.format(res)))[0].resolve())
        except: print('No pretrained model for this resolution')

    def update_res(self):
        self.current_resolution += 1
        self.current_width = 2 ** self.current_resolution

    def add_resolution(self):
        self.update_res()
        if self.current_resolution > 2:
            self.generator = tf.keras.models.load_model(self.get_model(self.model_folder,self.current_resolution), custom_objects={'leaky_relu': tf.nn.leaky_relu}, compile=True)
            self.generator.trainable = False

    def generate_latents(self,num_samples):
        latents = []
        for i in range(num_samples):
            latent = tf.random.normal((1, self.latent_size))
            latents.append(latent)
        return latents


import pandas as pd
from extensions import extensions
from initial_values.initial_values import be_data_columns_to_master_columns, year_dict
from datetime import datetime
from dateutil.relativedelta import relativedelta
from initial_values.initial_values import sap_user_status_cons_status_list, be_data_cons_status_list, sap_system_status_ban_list, operaton_status_translation, master_data_to_ru_columns, month_dict
import sqlite3
from openpyxl.utils.dataframe import dataframe_to_rows
import openpyxl

db = extensions.db

plug = 'plug'
date_time_plug = '1.1.2199'
date_time_plug = datetime.strptime(date_time_plug, '%d.%m.%Y')

# наименование итераций
iterations = {
  "Дата вывода 0 - итерация продления": "iteration_0",
  "Дата вывода 1 - итерация продления": "iteration_1",
  "Дата вывода 1 - итерация продления (корр 23.06.22)": "iteration_1_1",
  "Дата вывода 2 - итерация продления (корр 30.06.22)": "iteration_2"
}
iterations_list = ["operation_finish_date_iteration_0",	"operation_finish_date_iteration_1",	"operation_finish_date_iteration_2"]
iterations_dict = {
  # "operation_finish_date_iteration_0":"Дата вывода 0 - итерация продления",
  # "operation_finish_date_iteration_1":"Дата вывода 1 - итерация продления",
  "operation_finish_date_iteration_2":"Дата вывода 2 - итерация продления"
}

def read_be_2_eo_xlsx():
  # читаем загруженный файл
  be_eo_raw_data = pd.read_excel('uploads/be_eo_2_data.xlsx', sheet_name='be_eo_data', index_col = False)
  be_eo_data = be_eo_raw_data.rename(columns=be_data_columns_to_master_columns)

  # список колонок
  update_eo_column_list = list(be_eo_data.columns)

  be_eo_data['eo_code'] = be_eo_data['eo_code'].astype(str)
  be_eo_data['operation_finish_date_iteration_0'] = pd.to_datetime(be_eo_data['operation_finish_date_iteration_0'], format='%d.%m.%Y')
  be_eo_data['operation_finish_date_iteration_1'] = pd.to_datetime(be_eo_data['operation_finish_date_iteration_1'], format='%d.%m.%Y')
  be_eo_data['operation_finish_date_iteration_2'] = pd.to_datetime(be_eo_data['operation_finish_date_iteration_2'], format='%d.%m.%Y')
  be_eo_data['conservation_start_date'] = pd.to_datetime(be_eo_data['conservation_start_date'], format='%d.%m.%Y')
  be_eo_data['conservation_finish_date'] = pd.to_datetime(be_eo_data['conservation_finish_date'], format='%d.%m.%Y')
  be_eo_data['repair_start_date'] = pd.to_datetime(be_eo_data['repair_start_date'], format='%d.%m.%Y')
  be_eo_data['repair_finish_date'] = pd.to_datetime(be_eo_data['repair_finish_date'], format='%d.%m.%Y')

  # пишем заглушки в поля conservation_finish_date, repair_start_date, repair_finish_date

  be_eo_data['conservation_finish_date'].fillna(date_time_plug, inplace = True)

  # получаем данные из мастер-файла
  con = sqlite3.connect("database/datab.db")
  cursor = con.cursor()
  # sql = "SELECT * FROM eo_DB JOIN be_DB"
  sql = "SELECT \
  eo_DB.eo_code, \
  eo_DB.temp_eo_code, \
  eo_DB.temp_eo_code_status, \
  eo_DB.be_code, \
  eo_DB.head_type, \
  be_DB.be_description, \
  eo_DB.eo_class_code, \
  eo_class_DB.eo_class_description, \
  models_DB.eo_model_name, \
  models_DB.eo_category_spec, \
  models_DB.type_tehniki, \
  models_DB.marka_oborudovania, \
  models_DB.cost_center, \
  eo_DB.eo_model_id, \
  eo_DB.eo_description, \
  eo_DB.gar_no, \
  eo_DB.constr_type, \
  eo_DB.constr_type_descr, \
  eo_DB.operation_start_date, \
  eo_DB.expected_operation_period_years, \
  eo_DB.operation_finish_date_calc, \
  eo_DB.sap_planned_finish_operation_date, \
  eo_DB.operation_finish_date_sap_upd, \
  eo_DB.sap_system_status, \
  eo_DB.sap_user_status, \
  eo_DB.prodlenie_2022, \
  eo_DB.custom_eo_status \
  FROM eo_DB \
  LEFT JOIN models_DB ON eo_DB.eo_model_id = models_DB.eo_model_id \
  LEFT JOIN be_DB ON eo_DB.be_code = be_DB.be_code \
  LEFT JOIN eo_class_DB ON eo_DB.eo_class_code = eo_class_DB.eo_class_code \
  LEFT JOIN operation_statusDB ON eo_DB.expected_operation_status_code = operation_statusDB.operation_status_code"

  master_eo_df = pd.read_sql_query(sql, con)
  # подменяем временнные номера
  master_eo_df_temp_eo = master_eo_df.loc[master_eo_df['temp_eo_code_status']=='temp_eo_code']
  indexes_temp_eo = list(master_eo_df_temp_eo.index.values)
  # print(master_eo_df_temp_eo['temp_eo_code'])
  # print(list(master_eo_df_temp_eo.loc[indexes_temp_eo, ['temp_eo_code']]))
  master_eo_df.loc[indexes_temp_eo, ['eo_code']] = master_eo_df_temp_eo['temp_eo_code']
  # print(master_eo_df.loc[indexes_temp_eo, ['eo_code']])
  # print('должен быть список', master_eo_df.loc[indexes_temp_eo, ['eo_code']])

  master_eo_df = master_eo_df.loc[master_eo_df['head_type']=='head']
  master_eo_df['operation_start_date'] = pd.to_datetime(master_eo_df['operation_start_date'])

  master_eo_df['operation_start_date_month'] = ((master_eo_df['operation_start_date'] + pd.offsets.MonthEnd(0) - pd.offsets.MonthBegin(1)).dt.floor('d'))

  master_eo_df['sap_planned_finish_operation_date'] = pd.to_datetime(master_eo_df['sap_planned_finish_operation_date'])
  master_eo_df['operation_finish_date_sap_upd'] = pd.to_datetime(master_eo_df['operation_finish_date_sap_upd'])
  # джойним данные из файла с мастер-данными

  be_master_data = pd.merge(master_eo_df, be_eo_data, on='eo_code', how='left')

  #### обновление даты завершения из колонки Итерации
  for iteration, iteration_rus in iterations_dict.items():
    be_master_data[iteration] = pd.to_datetime(be_master_data[iteration])
    be_master_data[iteration].fillna(date_time_plug, inplace = True)

    be_master_data['operation_finish_date'] = be_master_data['operation_finish_date_sap_upd']
    be_master_data['operation_finish_date'] = be_master_data['operation_finish_date'].dt.date

    be_master_data['operation_finish_date'] = be_master_data['operation_finish_date_sap_upd']
    be_master_data['operation_finish_date'] = be_master_data['operation_finish_date'].dt.date

    be_master_data[iteration] = be_master_data[iteration].dt.date

    be_master_data_temp = be_master_data.loc[be_master_data[iteration]!=date_time_plug.date()]

    indexes = list(be_master_data_temp.index.values)

    be_master_data.loc[indexes, ['operation_finish_date']] = be_master_data_temp[iteration]

    be_master_data['operation_finish_date'] = pd.to_datetime(be_master_data['operation_finish_date'])

    be_master_data['operation_finish_date_month'] = ((be_master_data['operation_finish_date'] + pd.offsets.MonthEnd(0) - pd.offsets.MonthBegin(1)).dt.floor('d'))

    result_data_list = []
    i=0
    lenght = len(be_master_data)
    be_master_data['repair_start_date'].fillna(date_time_plug, inplace = True)
    be_master_data['repair_finish_date'].fillna(date_time_plug, inplace = True)
    be_master_data['os'].fillna(plug, inplace = True)
    # be_master_data.to_csv('temp_data/be_master_data_delete.csv')
    be_master_data = be_master_data.loc[be_master_data['custom_eo_status'] != 'delete']
    for row in be_master_data.itertuples():
      i=i+1
      # print("eo ", ", ", i, " из ", lenght)
      eo_code = getattr(row, 'eo_code')
      be_code = getattr(row, 'be_code')
      be_description = getattr(row, 'be_description')
      eo_class_code = getattr(row, 'eo_class_code')
      eo_class_description = getattr(row, 'eo_class_description')
      eo_model_name = getattr(row, 'eo_model_name')
      # eo_model_id = getattr(row, 'eo_model_id')
      eo_category_spec = getattr(row, 'eo_category_spec')
      type_tehniki = getattr(row, 'type_tehniki')
      marka_oborudovania = getattr(row, 'marka_oborudovania')

      eo_description = getattr(row, "eo_description")
      gar_no = getattr(row, "gar_no")
      constr_type = getattr(row, "constr_type")
      constr_type_descr = getattr(row, "constr_type_descr")
      os = getattr(row, "os")
      # operation_status_rus = getattr(row, "operation_status")
      sap_user_status = getattr(row, "sap_user_status")
      sap_system_status = getattr(row, "sap_system_status")
      cost_center = getattr(row, "cost_center")
      operation_status_from_file = getattr(row, "operation_status") # статус, полученный из файла

      operation_start_date = getattr(row, 'operation_start_date')
      operation_start_date_month = getattr(row, 'operation_start_date_month')
      expected_operation_period_years = getattr(row, 'expected_operation_period_years')
      operation_finish_date_calc = getattr(row, 'operation_finish_date_calc')
      sap_planned_finish_operation_date = getattr(row, 'sap_planned_finish_operation_date')
      operation_finish_date_sap_upd = getattr(row, 'operation_finish_date_sap_upd')
      # operation_finish_date_update_iteration = getattr(row, iteration)
      operation_finish_date = getattr(row, 'operation_finish_date')
      operation_finish_date_month = getattr(row, 'operation_finish_date_month')
      conservation_start_date = getattr(row, 'conservation_start_date')
      repair_start_date = getattr(row, 'repair_start_date')
      repair_finish_date = getattr(row, 'repair_finish_date')
      custom_eo_status = getattr(row, 'custom_eo_status')

      # сначала определяем статус ввода в эксплуатацию
      status_condition_dict = {
          "new":"Ввод нового",
          "on_balance":"На балансе",
          "conservation":"Консервация",
          "remake":"Переоборудование",
          "out":"План на вывод",
          "in_operation":"Эксплуатация",
          "out_of_order":"Неисправно"
        }
      for status_condition, status_condition_rus in status_condition_dict.items():

        if status_condition == "on_balance":
          if sap_system_status not in sap_system_status_ban_list:
            time_operation_point_date = operation_start_date
            # первый день в месяце
            time_operation_point_date = time_operation_point_date.replace(day=1)
            while time_operation_point_date <= operation_finish_date:
              temp_dict = {}
              temp_dict['eo_code'] = eo_code
              temp_dict['be_code'] = be_code

              temp_dict['be_description'] = be_description
              temp_dict['eo_class_code'] = eo_class_code
              temp_dict['eo_class_description'] = eo_class_description
              temp_dict['eo_model_name'] = eo_model_name
              temp_dict['eo_category_spec'] = eo_category_spec
              temp_dict['type_tehniki'] = type_tehniki
              temp_dict['marka_oborudovania'] = marka_oborudovania
              temp_dict['cost_center'] = cost_center
              if os != 'plug':
                temp_dict['os'] = os

              temp_dict['eo_description'] = eo_description
              temp_dict['sap_system_status'] = sap_system_status
              temp_dict['sap_user_status'] = sap_user_status
              temp_dict['operation_start_date'] = operation_start_date
              temp_dict['operation_finish_date'] = operation_finish_date
              temp_dict['operation_status'] = "На балансе"
              temp_dict['qty'] = 1
              temp_dict['На балансе'] = 1
              temp_dict['month_date'] = time_operation_point_date
              result_data_list.append(temp_dict)
              time_operation_point_date = time_operation_point_date + relativedelta(months=1)

        if status_condition == "in_operation":

          if sap_user_status not in sap_user_status_cons_status_list and \
          operation_status_from_file != "Консервация" and \
          sap_system_status not in sap_system_status_ban_list:
          # operation_start_date <= datetime.strptime('31.12.2023', '%d.%m.%Y') and \
          # operation_finish_date >= datetime.strptime('1.1.2022', '%d.%m.%Y'):
            time_operation_point_date = operation_start_date
            # первый день в месяце
            time_operation_point_date = time_operation_point_date.replace(day=1)

            # if eo_code == '100000065592':
            while time_operation_point_date <= operation_finish_date:
              temp_dict = {}
              temp_dict['eo_code'] = eo_code
              temp_dict['be_code'] = be_code
              temp_dict['be_description'] = be_description
              temp_dict['eo_class_code'] = eo_class_code
              temp_dict['eo_class_description'] = eo_class_description
              temp_dict['eo_model_name'] = eo_model_name
              temp_dict['eo_category_spec'] = eo_category_spec
              temp_dict['type_tehniki'] = type_tehniki
              temp_dict['marka_oborudovania'] = marka_oborudovania
              temp_dict['cost_center'] = cost_center
              if os != 'plug':
                temp_dict['os'] = os

              temp_dict['eo_description'] = eo_description
              temp_dict['sap_system_status'] = sap_system_status
              temp_dict['sap_user_status'] = sap_user_status
              temp_dict['operation_start_date'] = operation_start_date
              temp_dict['operation_finish_date'] = operation_finish_date
              temp_dict['operation_status'] = "Эксплуатация"
              temp_dict['qty'] = 1
              temp_dict['Эксплуатация'] = 1
              # print("time_operation_point_date: ", time_operation_point_date)
              # print("eo_code: ", eo_code, "operation_start_date: ", operation_start_date, "operation_finish_date: ", operation_finish_date)
              # print(time_operation_point)
              # if time_operation_point_date >=datetime.strptime('1.1.2022', '%d.%m.%Y'): #and \
              # time_operation_point_date <= datetime.strptime('31.12.2023', '%d.%m.%Y'):
                # print(time_operation_point)
              temp_dict['month_date'] = time_operation_point_date
              age = (time_operation_point_date - operation_start_date).days / 365.25
              temp_dict['age'] = age
              temp_dict['diagram_eo_count_in_operation'] = eo_code
              temp_dict['diagram_year'] = time_operation_point_date.year
                # print("time_operation_point_date: ", time_operation_point_date)
                # print("eo_code: ", eo_code, "operation_start_date: ", operation_start_date, "operation_finish_date: ", operation_finish_date)
                # print(temp_dict)
              # if repair_start_date == date_time_plug:
              #   result_data_list.append(temp_dict)
              if repair_start_date == date_time_plug:
                result_data_list.append(temp_dict)
              elif repair_start_date != date_time_plug and time_operation_point_date < repair_start_date:
                result_data_list.append(temp_dict)
              elif repair_start_date != date_time_plug and time_operation_point_date > repair_finish_date:
                result_data_list.append(temp_dict)
              else:
                if eo_code == '100000072785':
                  print(eo_code, "time_operation_point_date: ", time_operation_point_date, " repair_start_date: ", repair_start_date, " repair_finish_date:", repair_finish_date)
              #   and repair_start_date <= operation_finish_date and time_operation_point_date <= repair_start_date:
              #   result_data_list.append(temp_dict)
              # elif repair_start_date != date_time_plug and repair_start_date <= operation_finish_date and time_operation_point_date >= repair_finish_date:
              #   result_data_list.append(temp_dict)

              # result_data_list.append(temp_dict)
              time_operation_point_date = time_operation_point_date + relativedelta(months=1)

        if status_condition == "conservation":
          if (sap_user_status in sap_user_status_cons_status_list or \
          operation_status_from_file == "Консервация") and \
          sap_system_status not in sap_system_status_ban_list:
          # operation_start_date <= datetime.strptime('31.12.2023', '%d.%m.%Y') and \
          # operation_finish_date >= datetime.strptime('1.1.2022', '%d.%m.%Y'):
            time_operation_point_date = conservation_start_date
            conservation_finish_date = operation_finish_date
            # первый день в месяце
            time_operation_point_date = time_operation_point_date.replace(day=1)
            # if eo_code == '100000065592':
            while time_operation_point_date <= conservation_finish_date:
              temp_dict = {}
              temp_dict['eo_code'] = eo_code
              temp_dict['be_code'] = be_code
              temp_dict['be_description'] = be_description
              temp_dict['eo_class_code'] = eo_class_code
              temp_dict['eo_class_description'] = eo_class_description
              temp_dict['eo_model_name'] = eo_model_name
              temp_dict['eo_category_spec'] = eo_category_spec
              temp_dict['type_tehniki'] = type_tehniki
              temp_dict['marka_oborudovania'] = marka_oborudovania
              temp_dict['cost_center'] = cost_center
              if os != 'plug':
                temp_dict['os'] = os

              temp_dict['eo_description'] = eo_description
              temp_dict['sap_system_status'] = sap_system_status
              temp_dict['sap_user_status'] = sap_user_status
              temp_dict['operation_start_date'] = operation_start_date
              temp_dict['operation_finish_date'] = operation_finish_date
              temp_dict['conservation_start_date'] = conservation_start_date
              temp_dict['conservation_finish_date'] = conservation_finish_date
              temp_dict['operation_status'] = "Консервация"
              temp_dict['qty'] = 1
              temp_dict['Консервация'] = 1
              temp_dict['month_date'] = time_operation_point_date
              result_data_list.append(temp_dict)
              time_operation_point_date = time_operation_point_date + relativedelta(months=1)

        if status_condition == "new":
          # проверяем чтобы ео не была в консервации и в удаленных
          if sap_user_status not in sap_user_status_cons_status_list and \
          sap_system_status not in sap_system_status_ban_list:
            temp_dict = {}
            temp_dict['eo_code'] = eo_code
            temp_dict['be_code'] = be_code
            temp_dict['be_description'] = be_description
            temp_dict['eo_class_code'] = eo_class_code
            temp_dict['eo_class_description'] = eo_class_description
            temp_dict['eo_model_name'] = eo_model_name
            temp_dict['eo_category_spec'] = eo_category_spec
            temp_dict['type_tehniki'] = type_tehniki
            temp_dict['marka_oborudovania'] = marka_oborudovania
            temp_dict['cost_center'] = cost_center
            if os != 'plug':
                temp_dict['os'] = os

            temp_dict['eo_description'] = eo_description
            temp_dict['sap_system_status'] = sap_system_status
            temp_dict['sap_user_status'] = sap_user_status
            temp_dict['operation_start_date'] = operation_start_date
            temp_dict['operation_finish_date'] = operation_finish_date
            temp_dict['month_date'] = operation_start_date_month

            temp_dict['operation_status'] = "Ввод нового"
            temp_dict['qty'] = 1
            temp_dict['Ввод нового'] = 1

            result_data_list.append(temp_dict)

        elif status_condition == "out":
          if sap_system_status not in sap_system_status_ban_list:
            temp_dict = {}
            temp_dict['eo_code'] = eo_code
            temp_dict['be_code'] = be_code
            temp_dict['be_description'] = be_description
            temp_dict['eo_class_code'] = eo_class_code
            temp_dict['eo_class_description'] = eo_class_description
            temp_dict['eo_model_name'] = eo_model_name
            temp_dict['eo_category_spec'] = eo_category_spec
            temp_dict['type_tehniki'] = type_tehniki
            temp_dict['marka_oborudovania'] = marka_oborudovania
            temp_dict['cost_center'] = cost_center
            if os != 'plug':
                temp_dict['os'] = os

            temp_dict['eo_description'] = eo_description
            temp_dict['sap_system_status'] = sap_system_status
            temp_dict['sap_user_status'] = sap_user_status
            temp_dict['operation_start_date'] = operation_start_date
            temp_dict['operation_finish_date'] = operation_finish_date
            temp_dict['month_date'] = operation_finish_date_month

            temp_dict['operation_status'] = "План на вывод"
            temp_dict['qty'] = -1
            temp_dict['План на вывод'] = -1
            result_data_list.append(temp_dict)

        if status_condition == "out_of_order":
          if sap_system_status not in sap_system_status_ban_list:
            time_operation_point_date = operation_start_date
            # первый день в месяце
            time_operation_point_date = time_operation_point_date.replace(day=1)
            if repair_start_date != date_time_plug and repair_start_date <= operation_finish_date:
              while time_operation_point_date <= operation_finish_date:
                temp_dict = {}
                temp_dict['eo_code'] = eo_code
                temp_dict['be_code'] = be_code

                temp_dict['be_description'] = be_description
                temp_dict['eo_class_code'] = eo_class_code
                temp_dict['eo_class_description'] = eo_class_description
                temp_dict['eo_model_name'] = eo_model_name
                temp_dict['eo_category_spec'] = eo_category_spec
                temp_dict['type_tehniki'] = type_tehniki
                temp_dict['marka_oborudovania'] = marka_oborudovania
                temp_dict['cost_center'] = cost_center
                if os != 'plug':
                  temp_dict['os'] = os

                temp_dict['eo_description'] = eo_description
                temp_dict['sap_system_status'] = sap_system_status
                temp_dict['sap_user_status'] = sap_user_status
                temp_dict['operation_start_date'] = operation_start_date
                temp_dict['operation_finish_date'] = operation_finish_date
                temp_dict['operation_status'] = "Неисправен"
                temp_dict['qty'] = 1
                temp_dict['Неисправен'] = 1
                temp_dict['month_date'] = time_operation_point_date
                if time_operation_point_date >= repair_start_date and time_operation_point_date <= repair_finish_date:
                  result_data_list.append(temp_dict)
                time_operation_point_date = time_operation_point_date + relativedelta(months=1)

  iter_df_temp = pd.DataFrame(result_data_list)
  iter_df_temp['eo_count'] = iter_df_temp['eo_code']

  # режем результат по началу 22-го года
  iter_df_temp = iter_df_temp.loc[iter_df_temp['month_date']>=datetime.strptime('1.1.2022', '%d.%m.%Y')]
  # iter_df_temp = iter_df_temp.rename(columns=master_data_to_ru_columns)
  iter_df_temp.reset_index()

  list_of_be = list(set(iter_df_temp['be_code']))
  # list_of_be = [1100]

  for be_code in list_of_be:
    df = iter_df_temp.loc[iter_df_temp['be_code']==be_code]

    # iter_df_temp["month_date"] = iter_df_temp["month_date"].dt.strftime("%d.%m.%Y")

    wb = openpyxl.Workbook(write_only=True)
    ws = wb.create_sheet("Sheet1")

    i=0

    print(be_code, ": ", len(df))
    df = df.rename(columns=master_data_to_ru_columns)
    for r in dataframe_to_rows(df, index=False, header=True):
      i=i+1
      # print(i, " из ", lenght)
      ws.append(r)

    wb.save(f"temp_data/df_{be_code}.xlsx")

      # if eo_code == '100000061761':
      #   print(eo_code)


from functools import partial
from typing import Callable

import numpy as np
import rospy
from stable_baselines3.common.vec_env import VecNormalize
from supersuit.vector import ConcatVecEnv, MarkovVectorEnv
from supersuit.vector.sb3_vector_wrapper import SB3VecEnvWrapper

class MarkovVectorEnv_patched(MarkovVectorEnv):
    """Patched environment wrapper which creates the correct API for vector environments. Dones for dead agents are returned as True instead as False."""

    def step(self, actions):
        agent_set = set(self.par_env.agents)
        act_dict = {
            agent: actions[i]
            for i, agent in enumerate(self.par_env.possible_agents)
            if agent in agent_set
        }
        observations, rewards, dones, infos = self.par_env.step(act_dict)

        # adds last observation to info where user can get it
        if all(dones.values()):
            for agent, obs in observations.items():
                infos[agent]["terminal_observation"] = obs

        rews = np.array(
            [rewards.get(agent, 0) for agent in self.par_env.possible_agents],
            dtype=np.float32,
        )
        # we changed the default value to true instead of false
        dns = np.array(
            [dones.get(agent, True) for agent in self.par_env.possible_agents],
            dtype=np.uint8,
        )
        infs = [infos.get(agent, {}) for agent in self.par_env.possible_agents]

        if all(dones.values()):
            observations = self.reset()
        else:
            observations = self.concat_obs(observations)
        assert (
            self.black_death
            or self.par_env.agents == self.par_env.possible_agents
        ), "MarkovVectorEnv does not support environments with varying numbers of active agents unless black_death is set to True"
        return observations, rews, dns, infs

def vec_env_create(
    env_fn: Callable,
    agent_list_fn: Callable,
    num_robots: int,
    num_cpus: int,
    num_vec_envs: int,
    PATHS: dict,
) -> SB3VecEnvWrapper:
    """Function which vectorizes a given environment function in multiple parallel environments.

    Args:
        env_fn (Callable): Function that initializes an environment with wrappers
        agent_list_fn (Callable): Object containing the program arguments
        num_robots (int): Number of robots in the environment
        num_cpus (int): Maximal number of CPUs to use (Currently only process is used anyhow)
        num_vec_envs (int): Number of parallel environments to spawn
        PATHS (dict): Dictionary which holds hyperparameters for the experiment

    Returns:
        SB3VecEnvWrapper: Vectorized environments following the SB3 VecEnv API. Each each robot in an environment \
            poses as an environment in the vector.
    """
    env_list_fns = [
        partial(
            env_fn,
            ns=f"sim_{i}",
            num_agents=num_robots,
            agent_list_fn=agent_list_fn,
            PATHS=PATHS,
        )
        for i in range(1, num_vec_envs + 1)
    ]
    env = env_list_fns[0]()
    action_space = env.observation_space
    observation_space = env.observation_space
    metadata = env.metadata

    num_cpus = min(num_cpus, num_vec_envs)
    rospy.init_node("train_env", disable_signals=False, anonymous=True)
    vec_env = ConcatVecEnv(env_list_fns, observation_space, action_space)
    return SB3VecEnvWrapper(vec_env)


# -*- coding: utf-8 -*-
"""
Created on Wed Jun 13 15:41:54 2018

@author: usuario
"""

import pandas as pd
import numpy as np
from keras.models import load_model
from collections import Counter
import time
from datetime import datetime

def runClassifier (current_batch, clf):
    current_batch=np.array(current_batch)
    pred=clf.predict(current_batch)
    print(pred)
    class_pred=np.argmax(pred, axis=1)
    print(class_pred)
    counts=Counter(class_pred)
    print(counts)
    #voting
    v=list(counts.values())
    print(v)
    k=list(counts.keys())
    print(k)
    batch_fit=k[v.index(max(v))]
    print(batch_fit)
    return batch_fit

def classify(f_name, model, time_window, stride, batch_size=6, verbose=True):
    start_computing_time = time.time()

    output_sequence=[]
    time_stamp = []
    df = pd.read_csv(f_name)
    data=df [["x", "y", "z"]].values
    timestamp = df[["timestamp"]].values
    if (verbose):
        print('Data loaded.')
    clf=load_model(model)
    if (verbose):
        print('Model loaded.')

    offset=0

    current_batch=[]
    bc=0
    while (offset+time_window)<data.shape[0]:
        current_batch.append(data[offset:(offset+time_window)])

        if len(current_batch)==batch_size:
            #print (str(bc))
            if (verbose & (bc%500==0)):
                print('Progress (batches): '+ str(bc))
            bc+=1
            output_sequence.append(runClassifier(current_batch, clf))

            dt_object = datetime.fromtimestamp(timestamp[offset+time_window])
            time_stamp.append(dt_object.strftime("%d-%b-%Y (%H:%M:%S.%f)"))

            current_batch=[]
        offset+=time_window
    if len(current_batch)>0:
        output_sequence.append(runClassifier(current_batch, clf))
        dt_object = datetime.fromtimestamp(timestamp[offset])
        time_stamp.append(dt_object.strftime("%d-%b-%Y (%H:%M:%S.%f)"))

    total_computing_time = time.time() - start_computing_time
    print("computing time:", str(total_computing_time))

    return np.array(output_sequence), np.array(time_stamp)

f_name = '/home/khaosdev/Documentos/Sandro/Proyecto_Spark/ML_HAR/models/cyclingmodel1.csv'
model='/home/khaosdev/AnacondaProjects/Proyecto_Pulseras/clf_11.bin'

if __name__ == '__main__':
    outputs=classify(f_name, model, 1000, 1000)
    print(outputs)


import argparse

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--physics", help="physics marks")
    parser.add_argument("--chemistry", help="chemistry marks")
    parser.add_argument("--maths", help="maths marks")

    args = parser.parse_args()

    print(args.physics)
    print(args.chemistry)
    print(args.maths)

    print("Result:", (
        int(args.physics) + int(args.chemistry) + int(args.maths)
    ) / 3)

    # python3 cmd.py --physics 60 --chemistry 70 --maths 90


from django.urls import path

from . import views

app_name = 'main'

urlpatterns = [
    # not logged in
    path('', views.index, name="index"),
    path('search/', views.search, name="search"),

    # logged in
    path('home/', views.home, name="home"),
    path('post/', views.addWord, name="post"),
    path('results/', views.results, name="results"),
    path('dashboard/', views.dashbBoard, name="dashboard"),

    # authentication
    path('signup/', views.signup, name="signup"),
    path('login/', views.loginPage, name="login"),
    path('logout/', views.logoutUser, name="logout"),

]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: williamhadnett D00223305
"""

import pymongo
import os

os.chdir('/Users/williamhadnett/Documents/Data_Science/Data_Science_CA3_William_Hadnett')
import atlasCredentials
# =============================================================================
# Connect to MongoDB
# =============================================================================

connection = "mongodb+srv://"+atlasCredentials.username+":"+atlasCredentials.password+"@cluster0.gh4kb.mongodb.net/test?retryWrites=true&w=majority"
client = pymongo.MongoClient(connection)

mydb = client['test']

shopcol = mydb['websiteshop']

# =============================================================================
# Product Association
# =============================================================================

#Find top ten products for association analysis based on quantity purchased.
unwind = {'$unwind':'$Basket'}
group = {'$group': {'_id': '$Basket.StockCode', 'count': {'$sum': 1}}}
sort={'$sort':{'count':-1}}
limit={'$limit': 10}
top10 = list(shopcol.aggregate([unwind,group,sort,limit]))
print(top10)
# [{'_id': '85123A', 'count': 320}, {'_id': '22423', 'count': 211},
# {'_id': '22469', 'count': 182}, {'_id': '22834', 'count': 162},
# {'_id': '22111', 'count': 160}, {'_id': '22961', 'count': 160},
# {'_id': '21485', 'count': 155}, {'_id': '22470', 'count': 152},
# {'_id': '22113', 'count': 146}, {'_id': '22112', 'count': 143}]

# =============================================================================
# Product Association - Product 85123A - Product 21212 - (Benchmark)
# =============================================================================

group = {'$group': {'_id': 0, 'total': {'$sum': 1}}}
totalDocs = list(shopcol.aggregate([group]))
print(totalDocs)

# Support(x) =  # of transactions in which x appears/total transactions

query =  {'Basket.StockCode': '85123A'}
support85123A = shopcol.count_documents(query) / totalDocs[0]['total']
print(support85123A)
# Support 85123A bought:  0.153

query =  {'Basket.StockCode': {'$all': ['85123A', '21212']}}
supportBoth = shopcol.count_documents(query) / totalDocs[0]['total']
print(supportBoth)
# Support Both bought: 0.0105

#Confidence that 21212 will be bought when 85123A is bought.
#Conf(85123A -> 21212) = supp(85123A and 21212)/ supp(85123A)

conf = supportBoth / support85123A
print(conf)
# conf: 0.06862745098039216

# Lift
query =  {'Basket.StockCode': '21212'}
support21212 = shopcol.count_documents(query) / totalDocs[0]['total']
print(support21212)
# 0.056

#Life(85123A -> 21212) = supp(85123A and 21212)/ supp(85123A) * supp(21212)
lift = supportBoth / (support85123A * support21212)
print(lift)
# Lift: 1.2254901960784315
# So the support for 21212 is 0.004% more likely to bough if the basket contains
# product 85123A than in general.

# =============================================================================
# Product Association - Generalize formula for top 10
# =============================================================================

# The above support, confidence and lift will act as a bench mark to ensure that the
# calculates for the top ten are carried out correctly.

# This funciton is more general and can be applied to the top 10 as well as the
# entire database. However, please note that processing the entire database
# may take some time.
def calculateAssoication(mongoResponse):

    pairs = findPairs(mongoResponse)

    group = {'$group': {'_id': 0, 'total': {'$sum': 1}}}
    totalDocs = list(shopcol.aggregate([group]))

    for i in pairs:
        # Support(x) =  # of transactions in which x appears/total transactions
        # query =  {'Basket.StockCode': i[0]['_id']}
        supportItem1 = i[0]['count'] / totalDocs[0]['total']

        query =  {'Basket.StockCode': {'$all': [i[0]['_id'], i[1]['_id']]}}
        supportBoth = shopcol.count_documents(query) / totalDocs[0]['total']

        #Confidence that Item 1 will be bought when Item 2 is bought.
        #Conf(Item 1 -> Item 2) = supp(Item 1 and Item 2)/ supp(Item 1)

        # Lift
        query =  {'Basket.StockCode': i[1]['_id']}
        supportItem2 =  i[1]['count'] / totalDocs[0]['total']

        conf = supportBoth / supportItem1
        # The only metric that changes in regards to the inverse association
        # appears to be confidence as number of appearances remains the same for both
        # items individually and together in the same basket. This metric can be
        # gathered to display the inverse realtionship to the reader.
        inverseConf = supportBoth / supportItem2

        #Lift(Item 1 -> Item 2) = supp(Item 1 and Item 2)/ supp(Item1) * supp(Item2)
        lift = supportBoth / (supportItem1 * supportItem2)
        displayAssoication(supportItem1, supportBoth, supportItem2, conf, inverseConf, lift, i)

# Converting to a list of tuples using iterator
# https://stackoverflow.com/questions/23286254/how-to-convert-a-list-to-a-list-of-tuples
def findPairs(mongoResponse):
    it = iter(mongoResponse)
    pairs = list(zip(it, it))
    return pairs

def displayAssoication(support1, supportBoth, support2, conf, inverseConf, lift, i):

    print('Support for Item ',i[0]['_id'],': ',support1)
    print('Support Both: ',supportBoth)
    print('Support Item ',i[1]['_id'],': ',support2)
    print('Confidence: ',conf)
    print('Lift ',i[0]['_id'],' -> ',i[1]['_id'],': ',lift)

    print('\nSupport Item ',i[1]['_id'],': ',support2)
    print('Support Both: ',supportBoth)
    print('Support for Item ',i[0]['_id'],': ',support1)
    print('Confidence: ',inverseConf)
    print('Lift ',i[1]['_id'],' -> ',i[0]['_id'],': ',lift)
    print("\n")

calculateAssoication(top10)

# Output of Assoication Analysis of Top Ten Items
'''
Support for Item  85123A :  0.16
Support Both:  0.016
Support Item  22423 :  0.1055
Confidence:  0.1
Lift  85123A  ->  22423 :  0.9478672985781991

Support Item  22423 :  0.1055
Support Both:  0.016
Support for Item  85123A :  0.16
Confidence:  0.15165876777251186
Lift  22423  ->  85123A :  0.9478672985781991

Support for Item  22469 :  0.091
Support Both:  0.0085
Support Item  22834 :  0.081
Confidence:  0.09340659340659342
Lift  22469  ->  22834 :  1.1531678198344866

Support Item  22834 :  0.081
Support Both:  0.0085
Support for Item  22469 :  0.091
Confidence:  0.10493827160493828
Lift  22834  ->  22469 :  1.1531678198344866

Support for Item  22111 :  0.08
Support Both:  0.0105
Support Item  22961 :  0.08
Confidence:  0.13125
Lift  22111  ->  22961 :  1.640625

Support Item  22961 :  0.08
Support Both:  0.0105
Support for Item  22111 :  0.08
Confidence:  0.13125
Lift  22961  ->  22111 :  1.640625

Support for Item  21485 :  0.0775
Support Both:  0.0095
Support Item  22470 :  0.076
Confidence:  0.12258064516129032
Lift  21485  ->  22470 :  1.6129032258064517

Support Item  22470 :  0.076
Support Both:  0.0095
Support for Item  21485 :  0.0775
Confidence:  0.125
Lift  22470  ->  21485 :  1.6129032258064517

Support for Item  22113 :  0.073
Support Both:  0.017
Support Item  22112 :  0.0715
Confidence:  0.23287671232876717
Lift  22113  ->  22112 :  3.257016955647093

Support Item  22112 :  0.0715
Support Both:  0.017
Support for Item  22113 :  0.073
Confidence:  0.2377622377622378
Lift  22112  ->  22113 :  3.257016955647093
'''


import cv2 as cv
import os

def YOLO():

    dir = os.path.dirname(__file__)
    net = cv.dnn.readNetFromDarknet(dir + "/models/yolov3-tiny.cfg", dir + "/models/yolov3-tiny.weights")
    blob_options = {"scale": 1/255.0, "MeanSubtraction": (0, 0, 0)}
    labels = open(dir + "/data/coco2014.names").read().strip().split("\n")
    return net, blob_options, labels

def SSD():

    dir = os.path.dirname(__file__)
    net = cv.dnn.readNetFromTensorflow(dir + "/models/ssdlite_mobilenet_v2.pb", dir + "/models/ssdlite_mobilenet_v2.pbtxt")

    blob_options = {"scale": 1.0, "MeanSubtraction": (127.5, 127.5, 127.5)}

    labels = open(dir + "/data/coco2017.names").read().strip().split("\n")
    labels.insert(0, "unknown")
    return net, blob_options, labels

def FasterRCNN():

    dir = os.path.dirname(__file__)
    net = cv.dnn.readNetFromTensorflow(dir + "/models/faster_rcnn_inception_v2.pb", dir + "/models/faster_rcnn_inception_v2.pbtxt")

    blob_options = {"scale": 1, "MeanSubtraction": (127.5, 127.5, 127.5)}
    labels = open(dir + "/data/coco2017.names").read().strip().split("\n")
    return net, blob_options, labels


# -*- coding: utf-8 -*-
"""Example script to show how to use mcetl.launch_main_gui with defined DataSource objects.

@author: Donald Erb
Created on Aug 22, 2020

"""

import itertools

import mcetl
import numpy as np
import pandas as pd
from scipy import optimize

def offset_data(df, target_indices, calc_indices, excel_columns,
                first_row, offset=None, **kwargs):
    """Example CalculationFunction with named kwargs"""

    total_count = 0
    for i, sample in enumerate(calc_indices):
        for j, calc_col in enumerate(sample):
            if excel_columns is not None:
                y = df[target_indices[0][i][j]]
                y_col = excel_columns[target_indices[0][i][j]]
                calc = [
                    f'= {y_col}{k + first_row} + {offset * total_count}' for k in range(len(y))
                ]
                # use np.where(~np.isnan(y)) so that the calculation works for unequally-sized
                # datasets
                df[calc_col] = np.where(~np.isnan(y), calc, None)

            else:
                y_col = df[df.columns[target_indices[0][i][j]]]
                df[df.columns[calc_col]] = y_col + (offset * total_count)
            total_count += 1

    return df

def offset_normalized_data(df, target_indices, calc_indices, excel_columns,
                           offset=None, **kwargs):
    """Adds an offset to normalized data"""

    for i, sample in enumerate(calc_indices):
        for j, calc_col in enumerate(sample):
            y_col = df[df.columns[target_indices[0][i][j]]]

            if excel_columns is not None:
                df[calc_col] = y_col + f' + {offset * i}'
            else:
                df[calc_col] = y_col + (offset * i)

    return df

def normalize(df, target_indices, calc_indices, excel_columns, first_row, **kwargs):
    """Performs a min-max normalization to bound values between 0 and 1."""

    for i, sample in enumerate(calc_indices):
        for j, calc_col in enumerate(sample):
            if excel_columns is not None:
                y = df[target_indices[0][i][j]]
                y_col = excel_columns[target_indices[0][i][j]]
                end = y.count() + 2
                calc = [
                    (f'=({y_col}{k + first_row} - MIN({y_col}$3:{y_col}${end})) / '
                     f'(MAX({y_col}$3:{y_col}${end}) - MIN({y_col}$3:{y_col}${end}))')
                    for k in range(len(y))
                ]

                df[calc_col] = np.where(~np.isnan(y), calc, None)

            else:
                y_col = df.columns[target_indices[0][i][j]]
                min_y = df[y_col].min()
                max_y = df[y_col].max()

                df[calc_col] = (df[y_col] - min_y) / (max_y - min_y)

    return df

def split(df, target_indices, **kwargs):
    """Preprocess function that separates each entry where delta-x changes sign."""

    x_col = df[df.columns[target_indices[0]]].to_numpy()
    diff = np.diff(x_col)
    mask = np.where(np.sign(diff[1:]) != np.sign(diff[:-1]))[0] + 2 # +2 b/c diff is one less, and mask is one less than diff

    if len(mask) > 1:
        mask = np.array([mask[0], *mask[np.where(mask[1:] - mask[:-1] != 1)[0] + 1]]) # in case x[i] - x[i+1] = 0

    return np.array_split(df, mask)

def split_segments(df, target_indices, **kwargs):
    """
    Preprocess function that separates each entry based on the segment number.

    Also removes the segment column after processing since it is not needed
    in the final output.

    """

    segment_index = target_indices[0]
    segment_col = df[df.columns[segment_index]].to_numpy()
    mask = np.where(segment_col[:-1] != segment_col[1:])[0] + 1 # + 1 since mask loses one index

    output_dataframes = np.array_split(df, mask)

    for dataframe in output_dataframes:
        dataframe.drop(segment_index, 1, inplace=True)

    return output_dataframes

def derivative(df, target_indices, calc_indices, excel_columns, first_row, **kwargs):
    """Calculates the derivative."""

    for i, sample in enumerate(calc_indices):
        for j, calc_col in enumerate(sample):
            if excel_columns is not None:
                y = df[target_indices[1][i][j]]
                x_col = excel_columns[target_indices[0][i][j]]
                y_col = excel_columns[target_indices[1][i][j]]
                calc = [
                    f'= ({y_col}{k + first_row} - {y_col}{k + first_row - 1}) / ({x_col}{k + first_row} - {x_col}{k + first_row - 1})' for k in range(len(y))
                ]
                calc[0] = 0

                df[calc_col] = np.where(~np.isnan(y), calc, None)

            else:
                x = df[target_indices[0][i][j]].to_numpy()
                y = df[target_indices[1][i][j]].to_numpy()

                derivative = np.zeros(x.size)
                derivative[1:] = (y[1:] - y[0:-1]) / (x[1:] - x[0:-1])
                df[calc_col] = derivative

    return df

def pore_preprocessor(df, target_indices, **kwargs):
    """
    Sorts the dataframe according to the diameter.

    Easier to do for each individual data file rather than when each
    dataset is combined together.

    """

    return [df.sort_values(target_indices[0])]

def pore_analysis(df, target_indices, calc_indices, excel_columns, **kwargs):
    """
    Creates a histogram of pore sizes weighted by the pore area for each entry.

    Also computes the average pore diameter and the standard deviation of pore size.

    """

    if excel_columns is None and kwargs['processed'][0]:
        return df # to prevent processing twice
    elif excel_columns is not None:
        kwargs['processed'][0] = True

    max_pore_size = df[itertools.chain.from_iterable(target_indices[0])].max(numeric_only=True).max()
    pore_bins = np.arange(-kwargs['bin_size'][0], max_pore_size + kwargs['bin_size'][0],
                          kwargs['bin_size'][0])
    # in case the number of measured pores is less than the number of bins
    if pore_bins[1:].size > len(df):
        df = pd.concat((df, pd.DataFrame({'temp': pore_bins})), axis=1).drop('temp', axis=1)

    for i, sample in enumerate(calc_indices):
        for j in range(len(sample) // 10): # 10 calc columns per entry in each sample

            # d designates diameters, a designates areas
            d_index = target_indices[0][i][j]
            a_index = target_indices[1][i][j]

            nan_mask = (~np.isnan(df[d_index])) & (~np.isnan(df[a_index]))
            avg_pore_size = np.average(df[d_index][nan_mask], weights=df[a_index][nan_mask])
            area_histogram = np.histogram(df[d_index], pore_bins, weights=df[a_index])[0]
            norm_area_histogram = np.histogram(df[d_index], pore_bins,
                                               weights=df[a_index], density=True)[0] * kwargs['bin_size'][0]

            df[sample[1 + (j * 10)]] = pd.Series(pore_bins[1:])
            df[sample[2 + (j * 10)]] = pd.Series(np.histogram(df[d_index], pore_bins)[0])
            df[sample[3 + (j * 10)]] = pd.Series(area_histogram)
            df[sample[4 + (j * 10)]] = pd.Series(np.cumsum(area_histogram))
            df[sample[5 + (j * 10)]] = df[sample[3 + (j * 10)]] / kwargs['bin_size'][0]
            df[sample[6 + (j * 10)]] = pd.Series(np.cumsum(norm_area_histogram))
            df[sample[7 + (j * 10)]] = pd.Series(norm_area_histogram / kwargs['bin_size'][0])
            df[sample[8 + (j * 10)]] = pd.Series((
                'non-weighted', np.average(df[d_index][nan_mask]),
                'Area-weighted', avg_pore_size
            ))
            df[sample[9 + (j * 10)]] = pd.Series((
                '', np.std(df[d_index][nan_mask]),
                '', np.sqrt(np.average((df[d_index][nan_mask] - avg_pore_size)**2,
                                       weights=df[a_index][nan_mask]))
            ))

    return df

def pore_sample_summary(df, target_indices, calc_indices, excel_columns, **kwargs):
    """
    Creates a histogram of pore sizes weighted by the pore area for each sample.

    Also computes the average pore diameter and the standard deviation of pore size.

    """

    if excel_columns is None and kwargs['processed'][0]:
        return df # to prevent processing twice

    max_pore_size = df[itertools.chain.from_iterable(target_indices[0])].max(numeric_only=True).max()
    pore_bins = np.arange(-kwargs['bin_size'][0], max_pore_size + kwargs['bin_size'][0],
                          kwargs['bin_size'][0])

    for i, sample in enumerate(calc_indices):
        if not sample: # skip empty lists
            continue

        diameters = np.hstack([df[num][~np.isnan(df[num])] for num in target_indices[0][i]])
        areas = np.hstack([df[num][~np.isnan(df[num])] for num in target_indices[1][i]])

        avg_pore_size = np.average(diameters, weights=areas)
        area_histogram = np.histogram(diameters, pore_bins, weights=areas)[0]
        norm_area_histogram = np.histogram(diameters, pore_bins,
                                           weights=areas, density=True)[0] * kwargs['bin_size'][0]

        df[sample[0]] = pd.Series(pore_bins[1:])
        df[sample[1]] = pd.Series(np.histogram(diameters, pore_bins)[0])
        df[sample[2]] = pd.Series(area_histogram)
        df[sample[3]] = pd.Series(np.cumsum(area_histogram))
        df[sample[4]] = df[sample[2]] / kwargs['bin_size'][0]
        df[sample[5]] = pd.Series(np.cumsum(norm_area_histogram))
        df[sample[6]] = pd.Series(norm_area_histogram / kwargs['bin_size'][0])
        df[sample[7]] = pd.Series(('non-weighted', np.average(diameters),
                                   'Area-weighted', avg_pore_size))
        df[sample[8]] = pd.Series(('', np.std(diameters),
                                   '', np.sqrt(np.average((diameters - avg_pore_size)**2, weights=areas))))

    return df

def pore_dataset_summary(df, target_indices, calc_indices, excel_columns, **kwargs):
    """
    Summarizes the average pore size for each sample and its standard deviation.

    """

    if excel_columns is None and kwargs['processed'][0]:
        return df # to prevent processing twice

    # calc index is -1 since only the last dataframe is the dataset summary dataframe
    df[calc_indices[-1][0]] = pd.Series((f'Sample {num + 1}' for num in range(len(calc_indices[:-1]))))
    df[calc_indices[-1][1]] = pd.Series((df[indices[-2]][1] for indices in target_indices[0][:-1]))
    df[calc_indices[-1][2]] = pd.Series((df[indices[-1]][1] for indices in target_indices[0][:-1]))
    df[calc_indices[-1][3]] = pd.Series((df[indices[-2]][3] for indices in target_indices[0][:-1]))
    df[calc_indices[-1][4]] = pd.Series((df[indices[-1]][3] for indices in target_indices[0][:-1]))

    return df

def stress_model(strain, modulus):
    """
    Returns the linear estimate of the stress-strain curve using the strain and estimated modulus.

    Used for fitting data with scipy.

    Parameters
    ----------
    strain : array-like
        The array of experimental strain values, unitless (or with cancelled
        units, such as mm/mm).
    modulus : float
        The estimated elastic modulus for the data, with units of GPa (Pa * 10^9).

    Returns
    -------
    array-like
        The estimated stress data following the linear model, with units of Pa.

    """

    return strain * modulus * 1e9

def stress_strain_analysis(df, target_indices, calc_indices, excel_columns, **kwargs):
    """
    Calculates the mechanical properties from the stress-strain curve for each entry.

    Calculated properties include elastic modulus, 0.2% offset yield stress,
    ultimate tensile strength, and fracture strain.

    """

    if excel_columns is None and kwargs['processed'][0]:
        return df # to prevent processing twice

    empty_filler = 'N/A' if excel_columns is not None else None
    num_columns = 7 # the number of calculation columns per entry
    for i, sample in enumerate(calc_indices):
        for j in range(len(sample) // num_columns):
            strain_index = target_indices[0][i][j]
            stress_index = target_indices[1][i][j]
            nan_mask = (~np.isnan(df[strain_index])) & (~np.isnan(df[stress_index]))

            strain = df[strain_index].to_numpy()[nan_mask] / 100 # to convert from % to unitless
            stress = df[stress_index].to_numpy()[nan_mask] * 1e6 # to convert from MPa to Pa

            line_mask = (strain >= kwargs['lower_limit'][0]) & (strain <= kwargs['upper_limit'][0])
            modulus, covar = optimize.curve_fit(
                stress_model, strain[line_mask], stress[line_mask], p0=[80],
                method='trf', loss='soft_l1'
            )

            predicted_ultimate = np.nanmax(stress)
            uts_index = np.abs(stress - predicted_ultimate).argmin() + 1

            offset = stress - ((strain - 0.002) * modulus * 1e9) # 0.2% strain offset
            # using linear interpolation to get the exact crossing point of the offset and measured curves
            y0, y1 = (offset[offset > 0][-1], offset[offset <= 0][0])
            x0, x1 = (strain[offset > 0][-1], strain[offset <= 0][0])
            x_intercept = x0 - ((y0 * (x1 - x0)) / (y1 - y0))
            predicted_yield = float((x_intercept - 0.002) * modulus * 1e9)

            # predict fracture where stress[i] - stress[i + 1] is > 50 MPa
            try:
                predicted_fracture = 100 * strain[np.where(stress[:-1] - stress[1:] > 50e6)[0][0]]
            except IndexError: # fracture condition never reached
                predicted_fracture = 'N/A'

            df[sample[0 + (j * num_columns)]] = pd.Series(100 * np.log(1 + strain[:uts_index]))
            df[sample[1 + (j * num_columns)]] = pd.Series(stress[:uts_index] * (1 + strain[:uts_index]) / 1e6)
            df[sample[2 + (j * num_columns)]] = pd.Series(('Value', 'Standard Error'))
            df[sample[3 + (j * num_columns)]] = pd.Series((modulus[0], np.sqrt(np.diag(covar)[0])))
            df[sample[4 + (j * num_columns)]] = pd.Series((predicted_yield / 1e6, empty_filler))
            df[sample[5 + (j * num_columns)]] = pd.Series((predicted_ultimate / 1e6, empty_filler))
            df[sample[6 + (j * num_columns)]] = pd.Series((predicted_fracture, empty_filler))

    # prevents reprocessing the data
    kwargs['processed'][0] = True if excel_columns is not None else False

    return df

def tensile_sample_summary(df, target_indices, calc_indices, excel_columns, **kwargs):
    """
    Summarizes the mechanical properties for each sample.

    """

    if excel_columns is None and kwargs['processed'][0]:
        return df # to prevent processing twice

    num_cols = 7 # the number of calculation columns per entry from stress_strain_analysis
    for i, sample in enumerate(calc_indices):
        if not sample: # skip empty lists
            continue

        entries = [
            target_indices[0][i][j * num_cols:(j + 1) * num_cols] for j in range(len(target_indices[0][i]) // num_cols)
        ]

        df[sample[0]] = pd.Series(('Elastic Modulus (GPa)', 'Offset Yield Stress (MPa)',
                                   'Ultimate Tensile Strength (MPa)', 'Fracture Strain (%)'))
        df[sample[1]] = pd.Series(
            [np.mean([df[entry[3 + j]][0] for entry in entries if df[entry[3 + j]][0] != 'N/A']) for j in range(4)]
        )
        df[sample[2]] = pd.Series(
            [np.std([df[entry[3 + j]][0] for entry in entries if df[entry[3 + j]][0] != 'N/A']) for j in range(4)]
        )

    return df

def tensile_dataset_summary(df, target_indices, calc_indices, excel_columns, **kwargs):
    """
    Summarizes the mechanical properties for each dataset.

    """

    if excel_columns is None and kwargs['processed'][0]:
        return df # to prevent processing twice

    # calc index is -1 since only the last dataframe is the dataset summary dataframe
    df[calc_indices[-1][0]] = pd.Series([''] + [f'Sample {num + 1}' for num in range(len(calc_indices[:-1]))])
    df[calc_indices[-1][1]] = pd.Series(['Average'] + [df[indices[1]][0] for indices in target_indices[0][:-1]])
    df[calc_indices[-1][2]] = pd.Series(
        ['Standard Deviation'] + [df[indices[2]][0] for indices in target_indices[0][:-1]]
    )
    df[calc_indices[-1][3]] = pd.Series(['Average'] + [df[indices[1]][1] for indices in target_indices[0][:-1]])
    df[calc_indices[-1][4]] = pd.Series(
        ['Standard Deviation'] + [df[indices[2]][1] for indices in target_indices[0][:-1]]
    )
    df[calc_indices[-1][5]] = pd.Series(['Average'] + [df[indices[1]][2] for indices in target_indices[0][:-1]])
    df[calc_indices[-1][6]] = pd.Series(
        ['Standard Deviation'] + [df[indices[2]][2] for indices in target_indices[0][:-1]]
    )
    df[calc_indices[-1][7]] = pd.Series(['Average'] + [df[indices[1]][3] for indices in target_indices[0][:-1]])
    df[calc_indices[-1][8]] = pd.Series(
        ['Standard Deviation'] + [df[indices[2]][3] for indices in target_indices[0][:-1]]
    )

    return df

def carreau_model(shear_rate, mu_0, mu_inf, lambda_, n):
    """
    Estimates the Carreau model for viscosity.

    Used for fitting data using scipy.

    Parameters
    ----------
    shear_rate : array-like
        The experimental shear rate data, with units of 1/s.
    mu_0 : float
        The estimated viscosity at a shear rate of 0 1/s; units of Pa*s.
    mu_inf : float
        The estimated viscosity at infinite shear rate; units of Pa*s.
    lambda_ : float
        The reciprocal of the shear rate at which the material begins
        to flow in a non-Newtonian way; units of s.
    n : float
        The power law index for the material (1-n defines the slope of the
        curve of the non-Newtonian section of the log(viscosity) vs log(shear rate)
        curve); unitless.

    Returns
    -------
    array-like
        The estimated viscosity following the Carreau model, with units of Pa*s.

    """

    return mu_inf + (mu_0 - mu_inf) * (1 + (lambda_ * shear_rate)**2)**((n - 1) / 2)

def rheometry_analysis(df, target_indices, calc_indices, excel_columns, **kwargs):
    """
    Fits each data entry to the Carreau model and tabulates the results.

    """

    if excel_columns is None and kwargs['processed'][0]:
        return df # to prevent processing twice

    num_columns = 5 # the number of calculation columns per entry
    for i, sample in enumerate(calc_indices):
        for j in range(len(sample) // num_columns):
            shear_index = target_indices[0][i][j]
            viscosity_index = target_indices[1][i][j]
            nan_mask = (~np.isnan(df[shear_index])) & (~np.isnan(df[viscosity_index]))

            shear_rate = df[shear_index].to_numpy()[nan_mask]
            viscosity = df[viscosity_index].to_numpy()[nan_mask]

            # mu_0, mu_inf, lambda_, n
            initial_guess = (viscosity[0], viscosity[-1], 1, 0.2)
            bounds = ((1e-10, 1e-10, 1e-5, 1e-5), (1e10, 1e10, 1e5, 5))
            params, covariance = optimize.curve_fit(
                carreau_model, shear_rate, viscosity, p0=initial_guess,
                bounds=bounds, method='trf', loss='soft_l1'
            )
            # need to catch the following errors: ValueError('x0 is infeasible')
            predicted_viscosity = carreau_model(shear_rate, *params)
            r_sq = mcetl.fitting.r_squared(viscosity, predicted_viscosity, 4)[1]

            df[sample[1 + (j * num_columns)]] = pd.Series(predicted_viscosity)
            df[sample[2 + (j * num_columns)]] = pd.Series(
                ('\u03bc_0 (Pa*s)', '\u03bc_inf (Pa*s)',
                 '\u03bb, relaxation time (s)', 'n, power law index (unitless)',
                 '', 'Fit R\u00b2')
            )
            df[sample[3 + (j * num_columns)]] = pd.Series(list(params) + ['', r_sq])
            df[sample[4 + (j * num_columns)]] = pd.Series(np.sqrt(np.diag(covariance)))

    # prevents reprocessing the data
    kwargs['processed'][0] = True if excel_columns is not None else False

    return df

if __name__ == '__main__':

    # the kwargs for some functions; make a variable so it can be shared between Function objects;
    # uses lists as the values so that they can be permanently alterred
    pore_kwargs = {'bin_size': [5], 'processed': [False]}
    tensile_kwargs = {'lower_limit': [0.0015], 'upper_limit': [0.005], 'processed': [False]}

    # Definitions for the Function objects
    offset = mcetl.CalculationFunction(
        name='offset', target_columns='y', functions=offset_data,
        added_columns=1, function_kwargs={'offset': 1000}
    )
    normalize = mcetl.CalculationFunction('normalize', 'y', normalize, 1)
    offset_normalized = mcetl.CalculationFunction(
        'offset_normalized', 'normalize', offset_normalized_data, 'normalize', {'offset': 1}
    )
    delta_x_separator = mcetl.PreprocessFunction('delta_x_sep', 'temperature', split)
    segment_separator = mcetl.PreprocessFunction('segment_sep', 'segment', split_segments,
                                                 deleted_columns=['segment'])
    derivative_calc = mcetl.CalculationFunction('derivative', ['time', 'mass'], derivative, 1)
    pore_preprocess = mcetl.PreprocessFunction('pore_preprocess', 'diameter', pore_preprocessor)
    pore_histogram = mcetl.CalculationFunction(
        'pore_hist', ['diameter', 'area'], pore_analysis, 10, pore_kwargs
    )
    pore_sample_summation = mcetl.SummaryFunction(
        'pore_sample_sum', ['diameter', 'area'], pore_sample_summary, 9, pore_kwargs
    )
    pore_dataset_summation = mcetl.SummaryFunction(
        'pore_dataset_sum', ['pore_sample_sum'], pore_dataset_summary, 5,
        pore_kwargs, False
    )
    stress_analysis = mcetl.CalculationFunction(
        'tensile_test', ['strain', 'stress'], stress_strain_analysis, 7, tensile_kwargs
    )
    stress_sample_summary = mcetl.SummaryFunction(
        'tensile_sample_summary', ['tensile_test'], tensile_sample_summary, 3, tensile_kwargs
    )
    stress_dataset_summary = mcetl.SummaryFunction(
        'tensile_dataset_summary', ['tensile_sample_summary'], tensile_dataset_summary, 9,
        tensile_kwargs, False
    )
    rheometry_calc = mcetl.CalculationFunction(
        'rheology', ['shear rate', 'viscosity'], rheometry_analysis, 5, {'processed': [False]}
    )

    # Definitions for each data source
    xrd = mcetl.DataSource(
        name='XRD',
        column_labels=['2\u03B8 (\u00B0)', 'Intensity (Counts)', 'Offset Intensity (a.u.)'],
        functions=[offset],
        column_numbers=[1, 2],
        start_row=1,
        end_row=0,
        separator=',',
        xy_plot_indices=[0, 2],
        file_type='csv',
        num_files=1,
        unique_variables=['x', 'y'],
        entry_separation=1,
        sample_separation=2,
    )

    ftir = mcetl.DataSource(
        name='FTIR',
        column_labels=['Wavenumber (1/cm)', 'Absorbance (a.u.)', 'Normalized Absorbance (a.u.)'],
        functions=[normalize, offset_normalized],
        column_numbers=[0, 1],
        start_row=1,
        end_row=0,
        separator=',',
        xy_plot_indices=[0, 2],
        file_type='csv',
        num_files=1,
        unique_variables=['x', 'y'],
        entry_separation=1,
        sample_separation=2
    )

    raman = mcetl.DataSource(
        name='Raman',
        column_labels=['Raman Shift (1/cm)', 'Intensity (a.u.)', 'Normalized Intensity (a.u.)'],
        functions=[normalize, offset_normalized],
        column_numbers=[0, 1],
        start_row=0,
        end_row=0,
        separator='\t',
        xy_plot_indices=[0, 2],
        file_type='txt',
        num_files=1,
        unique_variables=['x', 'y'],
        entry_separation=1,
        sample_separation=2
    )

    tga = mcetl.DataSource(
        name='TGA',
        column_labels=['Temperature (\u00B0C)', 'Time (min)',
                       'Mass (%)', 'Mass Loss Rate (%/min)'],
        functions=[delta_x_separator, derivative_calc],
        column_numbers=[0, 1, 2],
        start_row=34,
        end_row=0,
        separator=';',
        xy_plot_indices=[0, 2],
        file_type='txt',
        num_files=1,
        unique_variables=['temperature', 'time', 'mass'],
        unique_variable_indices=[0, 1, 2],
        entry_separation=1,
        sample_separation=2
    )

    dsc = mcetl.DataSource(
        name='DSC',
        column_labels=['Temperature (\u00B0C)', 'Time (min)', 'Heat Flow, exo up (mW/mg)'],
        functions=[segment_separator],
        column_numbers=[0, 1, 2, 3],
        start_row=34,
        end_row=0,
        separator=';',
        xy_plot_indices=[1, 2],
        file_type='txt',
        num_files=1,
        unique_variables=['segment'],
        unique_variable_indices=[3],
        entry_separation=1,
        sample_separation=2
    )

    rheometry = mcetl.DataSource(
        name='Rheometry',
        column_labels=['Shear Stress (Pa)', 'Shear Rate (1/s)', 'Viscosity (Pa*s)',
                       'Time (s)', 'Temperature (\u00B0C)', '',
                       'Carreau Model Viscosity (Pa*s)', 'Carreau Model Variable',
                       'Value', 'Standard Error'],
        functions=[rheometry_calc],
        column_numbers=[0, 1, 2, 3, 4],
        start_row=167,
        end_row=0,
        separator='\t',
        xy_plot_indices=[1, 2],
        file_type='txt',
        num_files=1,
        unique_variables=['shear rate', 'viscosity'],
        unique_variable_indices=[1, 2],
        entry_separation=1,
        sample_separation=2
    )

    tensile = mcetl.DataSource(
        name='Tensile Test',
        column_labels=['Strain (%)', 'Stress (MPa)', 'Time (s)', 'Extension (mm)', 'Load (kN)',
                       'True Strain (%)', 'True Stress (MPa)',
                       '', 'Elastic Modulus (GPa)', 'Offset Yield Stress (MPa)',
                       'Ultimate Tensile Strength (MPa)', 'Fracture Strain (%)',
                       'Property', 'Average', 'Standard Deviation',
                       'Sample', 'Elastic Modulus (GPa)', '',
                       'Offset Yield Stress (MPa)', '',
                       'Ultimate Tensile Strength (MPa)', '',
                       'Fracture Strain (%)', ''],
        functions=[stress_analysis, stress_sample_summary, stress_dataset_summary],
        column_numbers=[4, 3, 0, 1, 2],
        start_row=6,
        end_row=0,
        separator=',',
        xy_plot_indices=[0, 1],
        file_type='txt',
        num_files=3,
        unique_variables=['stress', 'strain'],
        unique_variable_indices=[1, 0],
        entry_separation=2,
        sample_separation=3
    )

    pore_size = mcetl.DataSource(
        name='Pore Size Analysis',
        column_labels=['Measured Feret Diameters (\u03bcm)', 'Measured Areas (\u03bcm\u00b2)',
                       '', 'Histogram Diameter, D (\u03bcm)',
                       'Pore Count (#)', 'Area (\u03bcm\u00b2)',
                       'Cumulative Area, A (\u03bcm\u00b2)',
                       'Pore Size Distribution, dA/dD (\u03bcm\u00b2/\u03bcm)',
                       'Normalized Cumulative Area (\u03bcm\u00b2)',
                       'Normalized PSD, dA/dD (\u03bcm\u00b2/\u03bcm)',
                       'Average Diameter (\u03bcm)', 'Diameter Standard Deviation (\u03bcm)',
                       'Summarized Histogram Diameter, D (\u03bcm)',
                       'Summarized Pore Count (#)',
                       'Summarized Area (\u03bcm\u00b2)',
                       'Summarized Cumulative Area, A (\u03bcm\u00b2)',
                       'Summarized Pore Size Distribution, dA/dD (\u03bcm\u00b2/\u03bcm)',
                       'Summarized Normalized Cumulative Area (\u03bcm\u00b2)',
                       'Summarized Normalized PSD, dA/dD (\u03bcm\u00b2/\u03bcm)',
                       'Summarized Average Diameter (\u03bcm)',
                       'Summarized Diameter Standard Deviation (\u03bcm)',
                       'Sample', 'Average Diameter, non-weighted (\u03bcm)',
                       'Diameter Standard Deviation, non-weighted (\u03bcm)',
                       'Average Diameter, area-weighted (\u03bcm)',
                       'Diameter Standard Deviation, area-weighted (\u03bcm)'],
        functions=[pore_preprocess, pore_histogram,
                   pore_sample_summation, pore_dataset_summation],
        column_numbers=[4, 1],
        start_row=1,
        end_row=0,
        separator=',',
        xy_plot_indices=[3, 7],
        file_type='csv',
        num_files=3,
        unique_variables=['diameter', 'area'],
        unique_variable_indices=[0, 1],
        entry_separation=1,
        sample_separation=2
    )

    # For use in case you need to open arbitrary files without processing
    other = mcetl.DataSource('Other')

    # Put all DataSource objects in this tuple in order to use them
    data_sources = (xrd, ftir, raman, tga, dsc, rheometry, tensile, pore_size, other)

    #set dpi awareness so GUI is not blurry on Windows os
    mcetl.set_dpi_awareness()

    # Call the launch_main_gui function with data_sources as the input
    output = mcetl.launch_main_gui(data_sources)


from sqlalchemy.orm import sessionmaker

import sqlalchemy as db
from models.model import Base
from models.team_models import NCAATeam
from models.team_models import NBATeam
from models.oddshark_models import OddSharkNCAA
from models.oddshark_models import OddSharkNBA
from models.hasla_metrics_model import HaslaMetrics
from models.curated_picks_model import TeamRankingNCAA
from models.curated_picks_model import TeamRankingNBA
from models.curated_picks_model import PicksWiseNCAA
from models.curated_picks_model import PicksWiseNBA
from models.vegas_insider_model import VegasInsider
from models.espn_model import ESPNNCAAB
from models.betql_model import BetQL_NBA
from models.betql_model import BetQL_NCAA
from models.sportsinsights_model import SportsInsightsBETSIGNALS
from models.sportsinsights_model import SportsInsightsBESTBETS
import config
if config.SERVER_ENVIRONMENT:
    engine = db.create_engine(config.SERVER_DATABASE_URI)
else:
    engine = db.create_engine(config.LOCAL_DATABASE_URI)
connection = engine.connect()
if connection:
    print("Database opened successfully")
else:
    print("failed")
Session = sessionmaker(bind=engine)
session = Session()

def recreate_database():
    Base.metadata.drop_all(engine)
    Base.metadata.create_all(engine)
    print("Created All Tables")

def recreate_team_table(is_reset=0):
    if is_reset:
        Base.metadata.drop_all(engine, tables=[NCAATeam.__table__, NBATeam.__table__])
    Base.metadata.create_all(engine, tables=[NCAATeam.__table__, NBATeam.__table__])
    print("Created Team Tables")

def recreate_oddshark_table(is_reset=0):
    if is_reset:
        Base.metadata.drop_all(engine, tables=[OddSharkNBA.__table__, OddSharkNCAA.__table__])
    Base.metadata.create_all(engine, tables=[OddSharkNBA.__table__, OddSharkNCAA.__table__])
    print("Created OddShark Tables")

def recreate_hasla_metrics_table(is_reset=0):
    if is_reset:
        Base.metadata.drop_all(engine, tables=[HaslaMetrics.__table__])
    Base.metadata.create_all(engine, tables=[HaslaMetrics.__table__])
    print("Created hasla_metrics Tables")

def recreate_curated_picks_table(is_reset=0):
    if is_reset:
        Base.metadata.drop_all(engine, tables=[TeamRankingNCAA.__table__, TeamRankingNBA.__table__,
                                               PicksWiseNCAA.__table__, PicksWiseNBA.__table__])
    Base.metadata.create_all(engine, tables=[TeamRankingNCAA.__table__, TeamRankingNBA.__table__,
                                             PicksWiseNCAA.__table__, PicksWiseNBA.__table__])
    print("Created CuratedPicks Tables")

def recreate_espn_table(is_reset=0):
    if is_reset:
        Base.metadata.drop_all(engine, tables=[ESPNNCAAB.__table__])
    Base.metadata.create_all(engine, tables=[ESPNNCAAB.__table__])
    print("Created ESPN Tables")

def recreate_vegas_insider_table(is_reset=0):
    if is_reset:
        Base.metadata.drop_all(engine, tables=[VegasInsider.__table__])
    Base.metadata.create_all(engine, tables=[VegasInsider.__table__])
    print("Created VegasInsider Tables")

def recreate_betql_table(is_reset=0):
    if is_reset:
        Base.metadata.drop_all(engine, tables=[BetQL_NBA.__table__, BetQL_NCAA.__table__])
    Base.metadata.create_all(engine, tables=[BetQL_NBA.__table__, BetQL_NCAA.__table__])
    print("Created BetQL Tables")

def recreate_sportsinsights_table(is_reset=0):
    if is_reset:
        Base.metadata.drop_all(engine, tables=[SportsInsightsBETSIGNALS.__table__, SportsInsightsBESTBETS.__table__])
    Base.metadata.create_all(engine, tables=[SportsInsightsBETSIGNALS.__table__, SportsInsightsBESTBETS.__table__])
    print("Created SportsInsights Tables")

def close_connection():
    if connection:
        connection.close()
    if session:
        session.close()