# -*- coding: utf-8 -*-
"""
Created on Sun Mar 31 16:58:58 2019

@author: Ethan
"""

# -*- coding: utf-8 -*-
"""
Created on Thu Mar 21 11:25:12 2019

@author: Ethan
"""

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import statsmodels.api as sm
from sklearn import linear_model
import seaborn as sns
import pickle
import sys

#dataDir = '/dartfs-hpc/rc/lab/C/CMIG'
dataDir = 'e:/data/'

wxdata = 'era'

plotFigs = False

outageDaysOnly = False

dataset = 'all'

tempVar = 'txAvgSummer'

eData = {}
with open('eData.dat', 'rb') as f:
    eData = pickle.load(f)

entsoePlantData = eData['entsoePlantDataAll']
entsoeAgData = eData['entsoeAgDataAll']
nukeMatchData = eData['nukeMatchDataAll']
nukeAgData = eData['nukeAgDataAll']
nukePlantData = eData['nukePlantDataAll']

xtotal = []
if dataset == 'nuke' or dataset == 'all':
    xtotal.extend(nukeAgData[tempVar])

if dataset == 'entsoe' or dataset == 'all':
    xtotal.extend(entsoeAgData[tempVar])
xtotal = np.array(xtotal)

ytotal = []
if dataset == 'nuke' or dataset == 'all':
    ytotal.extend(nukeAgData['capacitySummer'])
if dataset == 'entsoe' or dataset == 'all':
    ytotal.extend(100*entsoeAgData['capacitySummer'])
ytotal = np.array(ytotal)

plantid = []
if dataset == 'nuke' or dataset == 'all':
    plantid.extend(nukeAgData['plantIds'])
if dataset == 'entsoe' or dataset == 'all':
    plantid.extend(entsoeAgData['plantIds'])
plantid = np.array(plantid)

plantMonths = []
if dataset == 'nuke' or dataset == 'all':
    plantMonths.extend(nukeAgData['plantMonths'])
if dataset == 'entsoe' or dataset == 'all':
    plantMonths.extend(entsoeAgData['plantMonths'])
plantMonths = np.array(plantMonths)

plantDays = []
if dataset == 'nuke' or dataset == 'all':
    plantDays.extend(nukeAgData['plantDays'])
if dataset == 'entsoe' or dataset == 'all':
    plantDays.extend(entsoeAgData['plantDays'])
plantDays = np.array(plantDays)

xtotalBool = []
if dataset == 'nuke' or dataset == 'all':
    xtotalBool.extend(nukeAgData[tempVar])
if dataset == 'entsoe' or dataset == 'all':
    xtotalBool.extend(entsoeAgData[tempVar])
xtotalBool = np.array(xtotalBool)

ytotalBool = []
if dataset == 'nuke' or dataset == 'all':
    ytotalBool.extend(nukeAgData['outagesBoolSummer'])
if dataset == 'entsoe' or dataset == 'all':
    ytotalBool.extend(entsoeAgData['outagesBoolSummer'])
ytotalBool = np.array(ytotalBool)

ind = np.where((ytotal <= 100.1) & (xtotal > 20))[0]
xtotal = xtotal[ind]
ytotal = ytotal[ind]

if outageDaysOnly:
    indOutages = np.where(ytotal<100)
    df = pd.DataFrame({'x':xtotal[indOutages], 'y':ytotal[indOutages]})

else:
    df = pd.DataFrame({'x':xtotal, 'y':ytotal})

plt.figure(figsize=(4,4))
plt.xlim([20,44])
plt.ylim([-25, 105])
sns.regplot(x='x', y='y', data=df, order=3, \
            scatter_kws={"color": [.5, .5, .5], "facecolor":[.75, .75, .75], "s":10, 'alpha':.25}, \
            line_kws={"color": [234/255., 49/255., 49/255.]})

sns.regplot(x='x', y='y', data=df, order=1, scatter=False, \
            line_kws={"color": [244/255., 153/255., 34/255.]})

#if outageDaysOnly:
#    plt.xlim([20, 44])
#    sns.regplot(x='x', y='y', data=df, lowess=True, scatter=False, \
#                line_kws={"color": [34/255., 171/255., 244/255.]})

plt.xlim([20,45])
plt.ylim([-25, 105])

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

plt.xlabel('Daily max temperature ($\degree$C)', fontname = 'Helvetica', fontsize=16)
plt.ylabel('Plant capacity (%)', fontname = 'Helvetica', fontsize=16)

plt.gca().set_xticks(range(20,45,4))
plt.gca().set_yticks(range(0,101,20))

x0,x1 = plt.gca().get_xlim()
y0,y1 = plt.gca().get_ylim()
plt.gca().set_aspect(abs(x1-x0)/abs(y1-y0))

#plt.title('July - August', fontname = 'Helvetica', fontsize=16)
if plotFigs:
    if outageDaysOnly:
        plt.savefig('nuke-eu-cap-reduction-outage-days.png', format='png', dpi=300, bbox_inches = 'tight', pad_inches = 0)
    else:
        plt.savefig('nuke-eu-cap-reduction-all-days.png', format='png', dpi=300, bbox_inches = 'tight', pad_inches = 0)

binstep = 4
bin_x1 = 20
bin_x2 = 44

bincounts = []
for t in range(bin_x1, bin_x2, binstep):
    if outageDaysOnly:
        bincounts.append(len(xtotal[(ytotal < 100) & (xtotal >= t) & (xtotal < t+binstep)]))
    else:
        bincounts.append(len(xtotal[(xtotal >= t) & (xtotal < t+binstep)]))

# plot hist of days in each temp bin
plt.figure(figsize=(6,1))
plt.xlim([19, 45])
#plt.ylim([0, 1])

plt.bar(range(bin_x1, bin_x2, binstep), bincounts, \
        facecolor = [.75, .75, .75], \
        edgecolor = [0, 0, 0], width = binstep, align = 'edge', \
        zorder=0)

plt.gca().set_xticks([])
plt.gca().set_xticklabels([])
if outageDaysOnly:
    plt.gca().set_yticks([0, 1500, 3000])
    plt.gca().set_yticklabels(['0', '1.5K', '3K'])
else:
    plt.gca().set_yticks([0, 10000, 20000])
    plt.gca().set_yticklabels(['0', '10K', '20K'])

plt.gca().set_yticks([])
plt.gca().set_yticklabels([])

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

if plotFigs:
    if outageDaysOnly:
        plt.savefig('temp-day-hist-only-outages-era.png', format='png', dpi=1000, bbox_inches = 'tight', pad_inches = 0)
    else:
        plt.savefig('temp-day-hist-all-days-era.png', format='png', dpi=1000, bbox_inches = 'tight', pad_inches = 0)

binstep = 20
bin_y1 = 0
bin_y2 = 105

bincounts = []
for t in range(bin_y1, bin_y2, binstep):
    if outageDaysOnly:
        bincounts.append(len(ytotal[(ytotal < 100) & (ytotal >= t) & (ytotal < t+binstep)]))
    else:
        bincounts.append(len(ytotal[(ytotal >= t) & (ytotal <= t+binstep)]))

# plot hist of days in each temp bin
plt.figure(figsize=(6,1))
plt.xlim([0, 100])
#plt.ylim([0, 1])

plt.bar(range(bin_y1, bin_y2, binstep), bincounts, \
        facecolor = [.75, .75, .75], \
        edgecolor = [0, 0, 0], width = binstep, align = 'edge', \
        zorder=0)

plt.gca().set_xticks([])
plt.gca().set_xticklabels([])
if outageDaysOnly:
    plt.gca().set_yticks([0, 2500, 5000])
    plt.gca().set_yticklabels(['0', '2.5K', '5K'])
else:
    plt.gca().set_yticks([0, 10000, 20000])
    plt.gca().set_yticklabels(['0', '10K', '20K'])

plt.gca().set_yticks([])
plt.gca().set_yticklabels([])

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

if plotFigs:
    if outageDaysOnly:
        plt.savefig('outage-day-hist-only-outages-era.png', format='png', dpi=1000, bbox_inches = 'tight', pad_inches = 0)
    else:
        plt.savefig('outage-day-hist-all-days-era.png', format='png', dpi=1000, bbox_inches = 'tight', pad_inches = 0)

dfLogistic = pd.DataFrame({'x':xtotalBool, 'y':ytotalBool})

binnedOutageData = []
binnedTx = []

binstep = 4
bin_x1 = 20
bin_x2 = 44

for i in range(entsoePlantData[tempVar].shape[0]):
    binnedOutageData.append([])
    for t in range(bin_x1, bin_x2, binstep):
        tempInds = np.where((entsoePlantData[tempVar][i] >= t) & (entsoePlantData[tempVar][i] < t+binstep))[0]
        if len(tempInds) > 0:
            binnedOutageData[i].append(np.nanmean(100 * entsoePlantData['outagesBoolSummer'][i, tempInds]))
        else:
            binnedOutageData[i].append(np.nan)

for i in range(nukeAgData['percCapacity'].shape[0]):
    binnedOutageData.append([])

    y = nukeAgData['percCapacity'][i]
    x = nukeMatchData['tx'][i,1:]

    if len(y) == len(x):
        y = y[nukeAgData['summerInds']]
        x = x[nukeAgData['summerInds']]
        nn = np.where((~np.isnan(y)) & (~np.isnan(x)))[0]

        # convert to boolean: 0 = no outage, > 0 = outage
        y = 100-y[nn]
        x = x[nn]
        y[y>0] = 100

        for t in range(bin_x1, bin_x2, binstep):
            tempInds = np.where((x >= t) & (x < t+binstep))[0]
            if len(tempInds) > 0:
                binnedOutageData[-1].append(np.nanmean(y[tempInds]))
            else:
                binnedOutageData[-1].append(np.nan)

binnedOutageData = np.array(binnedOutageData)
binnedOutageData = binnedOutageData / 100

boxy = []
boxyMeans = []
for c in range(binnedOutageData.shape[1]):
    boxy.append([])
    for i in range(binnedOutageData.shape[0]):
        if not np.isnan(binnedOutageData[i, c]):
            boxy[-1].append(binnedOutageData[i, c])
    boxyMeans.append(np.mean(boxy[-1]))

z = np.polyfit(range(bin_x1, bin_x2, binstep), np.nanmean(binnedOutageData, axis=0), 3)
p = np.poly1d(z)

plt.figure(figsize=(4,4))
#plt.bar(range(bin_x1, bin_x2, binstep), np.nanmean(binnedOutageData, axis=0),\
#        yerr = np.nanstd(binnedOutageData, axis=0)/2, \
#        facecolor = [.75, .75, .75], \
#        edgecolor = [0, 0, 0], width = 2, align = 'edge', \
#        error_kw=dict(lw=2, capsize=3, capthick=2), ecolor = [.25, .25, .25], zorder=0)

medianprops = dict(linestyle='-', linewidth=2, color='black')
meanpointprops = dict(marker='D', markeredgecolor='black',
                      markerfacecolor='red')
bplot = plt.boxplot(boxy, positions = range(22, 44, 4), showmeans=True, widths=4, sym='.', patch_artist=True, \
                    medianprops=medianprops, meanprops=meanpointprops, zorder=0)

colors = ['lightgray']
for patch in bplot['boxes']:
    patch.set_facecolor([.75, .75, .75])

plt.xlim([20, 44])
sns.regplot(x='x', y='y', data=dfLogistic, logistic=True, scatter=False, \
            line_kws={"color": [234/255., 49/255., 49/255.]})

#plt.plot(range(21, 44), p(range(20, 43)), "--", linewidth = 2, color = [234/255., 49/255., 49/255.])
plt.xlim([19, 45])
plt.ylim([0, .55])

plt.gca().set_xticks(range(20,45,4))
plt.gca().set_xticklabels(range(20,45,4))
plt.gca().set_yticks(np.arange(0,1.1,.20))
plt.gca().set_yticklabels(range(0,110,20))

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

plt.xlabel('Daily max temperature ($\degree$C)', fontname = 'Helvetica', fontsize=16)
plt.ylabel('Chance of outage (%)', fontname = 'Helvetica', fontsize=16)

x0,x1 = plt.gca().get_xlim()
y0,y1 = plt.gca().get_ylim()
plt.gca().set_aspect(abs(x1-x0)/abs(y1-y0))
if plotFigs:
    plt.savefig('nuke-eu-perc-plants-with-outages-%s.png'%wxdata, format='png', dpi=1000, bbox_inches = 'tight', pad_inches = 0)


import os

import matplotlib.pyplot as plt
import pandas as pd

df = pd.read_excel('branch_data.xlsx')

#print(df)

os.makedirs('plots/br_data', exist_ok=True)

for col1_idx, column1 in enumerate(df.columns):
    for col2_idx, column2 in enumerate(df.columns):
        if col1_idx < col2_idx:
            print(f'Generating {column1} to {column2} plot')
            fig, axes = plt.subplots(1, 1, figsize=(5, 5))
            axes.scatter(df[column1], df[column2], label=f'{column1} to {column2}', color='green', marker='x')
            axes.set_title(f'{column1} to {column2}')
            axes.set_xlabel(column1)
            axes.set_ylabel(column2)
            axes.legend()
            plt.savefig(f'plots/br_data/br_data_{column1}_{column2}_scatter.png', dpi=300)
            plt.close(fig)

plt.close()


from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import tensorflow as tf
from collections import OrderedDict

from .utils import get_from_module
from .utils import process_params

LOSSES = tf.GraphKeys().LOSSES

__all__ = ['add_loss',
           'get_losses',
           'get_regularization_losses',
           'get_total_loss',
           'l1_loss',
           'l2_loss',
           'get',
           'process_parameters']

def add_loss(loss):
    """Adds an externally defined loss to collection of losses.

    Parameters
    ----------
    loss: A loss `Tensor`.
    """
    tf.add_to_collection(LOSSES, loss)

def get_losses(scope=None):
    """Gets the list of loss variables stored in the collection of losses.

    Parameters
    ----------
    scope: an optional scope for filtering the losses to return.

    Returns
    -------
    a list of loss variables.
    """
    return tf.get_collection(LOSSES, scope)

def get_regularization_losses(scope=None):
    """Gets the regularization losses.

    Parameters
    ----------
    scope: an optional scope for filtering the losses to return.

    Returns
    -------
    A list of loss variables.
    """
    return tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES, scope)

def get_total_loss(add_regularization_losses=True, name="total_loss"):
    """Returns a tensor whose value represents the total loss.

    Notice that the function adds the given losses to the regularization losses.

    Parameters
    ----------
    add_regularization_losses: A boolean indicating whether or not to use the
        regularization losses in the sum.
    name: The name of the returned tensor.

    Returns
    -------
    A `Tensor` whose value represents the total loss.

    Raises
    ------
    ValueError: if `losses` is not iterable.
    """
    losses = get_losses()
    if add_regularization_losses:
        losses += get_regularization_losses()
    return tf.add_n(losses, name=name)

def l1_loss(tensor, weight=1.0, scope=None):
    """Define a L1Loss, useful for regularize, i.e. lasso.

    Parameters
    ----------
    tensor: tensor to regularize.
    weight: scale the loss by this factor.
    scope: Optional scope for op_scope.

    Returns
    -------
    the L1 loss op.
    """
    with tf.op_scope([tensor], scope, 'L1Loss'):
        weight = tf.convert_to_tensor(weight,
                                      dtype=tensor.dtype.base_dtype,
                                      name='loss_weight')
        loss = tf.mul(weight, tf.reduce_sum(tf.abs(tensor)), name='value')
        add_loss(loss)
        return loss

def l2_loss(tensor, weight=1.0, scope=None):
    """Define a L2Loss, useful for regularize, i.e. weight decay.

    Parameters
    ----------
    tensor: tensor to regularize.
    weight: an optional weight to modulate the loss.
    scope: Optional scope for op_scope.

    Returns
    -------
    the L2 loss op.
    """
    with tf.op_scope([tensor], scope, 'L2Loss'):
        weight = tf.convert_to_tensor(weight,
                                      dtype=tensor.dtype.base_dtype,
                                      name='loss_weight')
        loss = tf.mul(weight, tf.nn.l2_loss(tensor), name='value')
        add_loss(loss)
        return loss

def _get_defaults():
    return {
        'l1': OrderedDict([('scale', 0.01), ('scope', None)]),
        'l2': OrderedDict([('scale', 0.01), ('scope', None)]),
        # 'l1_l2': OrderedDict([('scale_l1', 0.01), ('scale_l2', 0.01), ('scope', None)]),
        # 'sum': OrderedDict([('regularizer_list', None), ('scope', None)])
    }

def get(name, kwargs=None):
    fn_dict = {
        'l1': l1_loss,
        'l2': l2_loss,
        # 'l1_l2': l1_l2_loss,
        # 'sum': sum_loss
    }

    return get_from_module(name, fn_dict, _get_defaults(), True, True, kwargs)

def process_parameters(name, kwargs):
    return process_params(name, kwargs, _get_defaults())


import asyncio
from twscrape import API, gather
from twscrape.logger import set_log_level
import pandas as pd
from datetime import datetime
import locale
#from scrapper.parse_args import parse_args
from parse_args import parse_args

async def main():

    args = parse_args()

    search = f'"{args.search}"' if args.search else args.search
    lang = f'lang:{args.lang}' if args.lang else ''
    since = f'since:{args.since}' if args.since else ''
    until = f'until:{args.until}' if args.until else ''

    if not search:
        raise ValueError('you must specify a search prompt to scrape')

    # Validar que la fecha de 'since' sea menor o igual a 'until'
    if args.since and args.until:
        since_date = datetime.strptime(args.since, '%Y-%m-%d')
        until_date = datetime.strptime(args.until, '%Y-%m-%d')
        if since_date > until_date:
            raise ValueError('The "since" date must be less than or equal to the "until" date.')

    api = API()

    q = ' '.join([search, lang, since, until])
    print(q)

    data = []
    async for tweet in api.search(q, limit = args.limit):
        data.append([
            tweet.id, tweet.date, tweet.rawContent, tweet.user.username,
            tweet.user.location, tweet.lang, tweet.hashtags,
            tweet.likeCount, tweet.replyCount, tweet.retweetCount,
            ])

    data = pd.DataFrame(data, columns = [
        'id', 'date', 'tweet', 'username',
        'user_location', 'language', 'hashtags',
        'likeCount', 'replyCount', 'retweetCount'
        ])

    data.to_csv(f'data/{to_camel_case(args.search)}.csv', encoding = 'utf-8-sig', index = False)

def format_date(fecha_str):

    locale.setlocale(locale.LC_TIME, 'es_ES.UTF-8')

    # Analizar la fecha en el formato dado
    fecha = datetime.strptime(fecha_str, "%Y-%m-%d %H:%M:%S+00:00")

    # Crear una cadena en el formato deseado
    #fecha_formateada = fecha.strftime("%d de %B de %Y")
    fecha_formateada = fecha.date()

    return fecha_formateada

def to_camel_case(text):
    # Divide el texto en palabras separadas por espacios
    words = text.split()

    # Capitaliza la primera letra de cada palabra (excepto la primera palabra)
    camel_words = [words[0].lower()] + [word.capitalize() for word in words[1:]]

    # Une las palabras para formar el Camel Case
    camel_case = ''.join(camel_words)

    return camel_case

def export_tweets():

    args = parse_args()

    df = pd.read_csv(f'data/{to_camel_case(args.search)}.csv')
    df['prefix'] = [f'Tweet {tweet}: ' for tweet in range(df.shape[0])]

    df['tweet'] = df['tweet'].apply(lambda x: ' '.join(x.split()) + '. ')

    df['user_location'] = df['user_location'].fillna('')
    df['user_location'] = df['user_location'].apply(lambda x: f'Ubicación: {x}. ')

    df['date'] = df['date'].apply(lambda x: f'Fecha: {format_date(x)}. ')
    df['tweet'] = df['prefix'] + df['tweet'] + df['date'] + df['user_location']

    megadoc = '\n'.join(df['tweet'])
    with open(f'documents/{to_camel_case(args.search)}_corpus.txt', 'w') as file:
        file.write(megadoc)

if __name__ == "__main__":
    # scrape tweets
    asyncio.run(main())

    # export tweets to .txt
    export_tweets()

    # montar aplicacion con gradio para buscar tweets + QA con LLM?


from django.shortcuts import render,HttpResponse,redirect
from.utils import send_mail_to_user
from .forms import *
import csv
import re
from django.core.mail import EmailMessage
def index(request):
    return render(request,'home.html')

def send_mail(request):
    message = request.GET.get('message')
    print(message)
    if message is not None:
        send_mail_to_user(['m.sujanpaudel256@gmail.com'],'Mail from Django',"THis is mail from django app and message is {}".format(message))
        print('message sent -  {}'.format(message))
        return HttpResponse("Mail sent to m.sujanpaudel256@gmail.com ")
    return render(request,'home.html')

def send_single_mail(request):
    form = SingleEmailForm()
    if request.method == 'POST':
        form = SingleEmailForm(request.POST)
        if form.is_valid():
            print(form.cleaned_data)
            email = form.cleaned_data.get('email')
            subject = form.cleaned_data.get('subject')
            body = form.cleaned_data.get('message')

            from_email = 'poudelaarav@gmail.com'  # Replace with your email address
            email_message = EmailMessage(subject, body,from_email, [email])
            email_message.send()

            # return render(request,'single_mail.html',{ 'form':form,'message_list':body})
            return redirect('send_single_mail')
    return render(request,'single_mail.html',{'form':form})

def send_bulk_mail(request):
    form = EmailTemplateForm()
    if request.method == 'POST':
        form = EmailTemplateForm(request.POST,request.FILES)
        if form.is_valid():
            print(form.cleaned_data)
            csv_file = form.cleaned_data['file']
            # csv_file = request.FILES['csv_file']

            reader = csv.reader(csv_file.read().decode('utf-8').splitlines())
            next(reader)  # Skip the header row if it exists

            messages = []
            template = form.cleaned_data.get('message_template')
            subject = form.cleaned_data.get('subject')
            only_gen = form.cleaned_data.get('only_gen')
            for row in reader:
                if len(row)>=4:
                    email = row[0]
                    col1= row[1]
                    col2 = row[2]
                    col3 = row[3]
                else:
                    email = row[0]
                    col1= row[1]
                    col2 = row[2]
                    col3 = ""

                # Template message
                # Fill in the template with actual values
                dynamic_message = re.sub(r'\{col1\}', col1, template)
                dynamic_message = re.sub(r'\{col2\}', col2, dynamic_message)
                dynamic_message = re.sub(r'\{col3\}',col3,dynamic_message)

                messages.append(dynamic_message)

                subject = subject
                body = dynamic_message
                from_email = 'poudelaarav@gmail.com'  # Replace with your email address

                email_message = EmailMessage(subject, body, from_email, [email])
                if not only_gen:
                    email_message.send()
            # Join all the messages with a line break
            final_message = '\n'.join(messages)
            print(final_message)
            # Return the final message as a response
            # return
            return render(request,'email_template.html',{'form':form,'message_list':messages,'only_gen':only_gen})

    return render(request,'email_template.html',{'form':form})


import bpy
from bpy.props import PointerProperty

from terrain_eroder.panels.creator_panel import CreatorPanel, AnimateObjectOperator
from terrain_eroder.panels.erosion_panel import ErosionPanel
from terrain_eroder.operators.create_terrain_operator import CreateTerrainOperator
from terrain_eroder.operators.erosion_operator import ErosionOperator
from terrain_eroder.properties.terrain_properties import TerrainProperties
from terrain_eroder.properties.erosion_properties import ErosionProperties

bl_info = {
    "name": "Terrain Eroder",
    "author": "Peter Urgoš",
    "version": (1, 0, 1),
    "blender": (2, 93, 0),
    "location": "View3D > Sidebar > Eroder",
    "description": "Add-on for terrain erosion",
    "category": "3D View"
}

# Classes inheriting from Blender that need to be (de-)activated
classes = (
    TerrainProperties,
    ErosionProperties,
    CreateTerrainOperator,
    AnimateObjectOperator,
    ErosionOperator,
    CreatorPanel,
    ErosionPanel
)

def register():
    for cls in classes:
        bpy.utils.register_class(cls)
    bpy.types.Scene.erosion_properties = PointerProperty(type=ErosionProperties, name="Erosion Properties", description="Properties of erosion")
    bpy.types.Scene.terrain_properties = PointerProperty(type=TerrainProperties, name="Terrain Properties", description="Properties for terrain creation")

def unregister():
    for cls in reversed(classes):
        bpy.utils.unregister_class(cls)

if __name__ == "__main__":
    register()


import logging
import os
from argparse import ArgumentParser
from typing import Optional

from bsf.analysis import Analysis, Stage
from bsf.executables.collection import RunnableStepCollectionPruneFastq
from bsf.ngs import Collection, Sample, PairedReads, Reads
from bsf.procedure import FilePath, ConsecutiveRunnable
from bsf.process import Command, RunnableStep, RunnableStepJava, RunnableStepMakeDirectory
from bsf.standards import Configuration, JavaArchive

module_logger = logging.getLogger(name=__name__)

class FilePathTrimmomaticReadGroup(FilePath):
    """The :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticReadGroup` class models
    read group-specific Trimmomatic files.

    :ivar output_directory: An output directory path.
    :type output_directory: str
    :ivar trim_log_tsv: A Trimmomatic trim log :emphasis:`Tab-Separated Value` (TSV) file path.
    :type trim_log_tsv: str
    :ivar summary_tsv: A summary :emphasis:`Tab-Separated Value` (TSV) file path.
    :type summary_tsv: str
    :ivar coverage_png: A coverage :emphasis:`Portable Network Graphics` (PNG) file path.
    :type coverage_png: str
    :ivar frequency_png: A frequency :emphasis:`Portable Network Graphics` (PNG) file path.
    :type frequency_png: str
    :ivar surviving_png: A surviving :emphasis:`Portable Network Graphics` (PNG) file path.
    :type surviving_png: str
    :ivar reads_1p: A First Reads paired file path.
    :type reads_1p: str | None
    :ivar reads_1u: First Reads unpaired file path.
    :type reads_1u: str | None
    :ivar reads_2p: Second Reads paired file path.
    :type reads_2p: str | None
    :ivar reads_2u: Second Reads unpaired file path.
    :type reads_2u: str | None
    """

    def __init__(self, prefix: str) -> None:
        """Initialise a :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticReadGroup` object.

        :param prefix: A Python :py:class:`str` prefix representing a :py:attr:`bsf.procedure.Runnable.name` attribute.
        :type prefix: str
        """
        super(FilePathTrimmomaticReadGroup, self).__init__(prefix=prefix)

        self.output_directory = prefix
        # Automatic GNU Zip-compression of trim log files does not work.
        self.trim_log_tsv = prefix + '_trim_log.tsv'
        self.trim_log_tsv_gz = prefix + '_trim_log.tsv.gz'
        self.summary_tsv = prefix + '_summary.tsv'  # Defined by the R script.
        self.coverage_png = prefix + '_coverage.png'  # Defined by the R script.
        self.frequency_png = prefix + '_frequency.png'  # Defined by the R script.
        self.surviving_png = prefix + '_surviving.png'  # Defined by the R script.
        self.reads_1p = None
        self.reads_1u = None
        self.reads_2p = None
        self.reads_2u = None

        return

class FilePathTrimmomaticProject(FilePath):
    """The :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticProject` class models
    project-specific Trimmomatic files.

    :ivar output_directory: An output directory path.
    :type output_directory: str
    :ivar sas_path_old: An old Sample Annotation Sheet file path.
    :type sas_path_old: str
    :ivar sas_path_new: A new Sample Annotation Sheet file path.
    :type sas_path_new: str
    """

    def __init__(self, prefix: str, prefix_analysis: str, project_name: str) -> None:
        """Initialise a :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticProject` object.

        :param prefix: A Python :py:class:`str` prefix representing a :py:attr:`bsf.procedure.Runnable.name` attribute.
        :type prefix: str
        :param prefix_analysis: A :py:attr:`bsf.analysis.Analysis.prefix` attribute.
        :type prefix_analysis: str
        :param project_name: A project name.
        :type project_name: str
        """
        super(FilePathTrimmomaticProject, self).__init__(prefix=prefix)

        self.output_directory = prefix
        self.sas_path_old = '_'.join((project_name, prefix_analysis, 'original.csv'))
        self.sas_path_new = '_'.join((project_name, prefix_analysis, 'samples.csv'))

        return

class Trimmomatic(Analysis):
    """The :py:class:`bsf.analyses.trimmomatic.Trimmomatic` class represents the logic to run the Trimmomatic analysis.

    :ivar adapter_path: An adapter definition file path.
    :type adapter_path: str | None
    :ivar trimming_step_pe_list: A colon-separated list of Trimmomatic steps for paired-end data.
    :type trimming_step_pe_list: list[str] | None
    :ivar trimming_step_se_list: A colon-separated list of Trimmomatic steps for single-end data.
    :type trimming_step_se_list: list[str] | None
    :ivar java_archive_trimmomatic: A Trimmomatic tool :emphasis:`Java Archive` (JAR) file path.
    :type java_archive_trimmomatic: str | None
    """

    name = 'Trimmomatic Analysis'
    prefix = 'trimmomatic'

    @classmethod
    def get_stage_name_read_group(cls) -> str:
        """Get a particular :py:attr:`bsf.analysis.Stage.name` attribute.

        :return: A :py:attr:`bsf.analysis.Stage.name` attribute.
        :rtype: str
        """
        return '_'.join((cls.prefix, 'read_group'))

    @classmethod
    def get_stage_name_summary(cls) -> str:
        """Get a particular :py:attr:`bsf.analysis.Stage.name` attribute.

        :return: A :py:attr:`bsf.analysis.Stage.name` attribute.
        :rtype: str
        """
        return '_'.join((cls.prefix, 'summary'))

    @classmethod
    def get_stage_name_project(cls) -> str:
        """Get a particular :py:attr:`bsf.analysis.Stage.name` attribute.

        :return: A :py:attr:`bsf.analysis.Stage.name` attribute.
        :rtype: str
        """
        return '_'.join((cls.prefix, 'project'))

    @classmethod
    def get_prefix_read_group(cls, read_group_name: str) -> str:
        """Get a Python :py:class:`str` prefix representing a :py:class:`bsf.procedure.Runnable` object.

        :param read_group_name: A read group name.
        :type read_group_name: str
        :return: A Python :py:class:`str` (prefix)  object representing a :py:class:`bsf.procedure.Runnable` object.
        :rtype: str
        """
        return '_'.join((cls.get_stage_name_read_group(), read_group_name))

    @classmethod
    def get_prefix_summary(cls) -> str:
        """Get a Python :py:class:`str` prefix representing a :py:class:`bsf.procedure.Runnable` object.

        :return: A Python :py:class:`str` (prefix)  object representing a :py:class:`bsf.procedure.Runnable` object.
        :rtype: str
        """
        return cls.get_stage_name_summary()

    @classmethod
    def get_prefix_project(cls) -> str:
        """Get a Python :py:class:`str` prefix representing a :py:class:`bsf.procedure.Runnable` object.

        :return: A Python :py:class:`str` (prefix)  object representing a :py:class:`bsf.procedure.Runnable` object.
        :rtype: str
        """
        return cls.get_stage_name_project()

    @classmethod
    def get_file_path_read_group(cls, read_group_name: str) -> FilePathTrimmomaticReadGroup:
        """Get a :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticReadGroup` object.

        :param read_group_name: A read group name.
        :type read_group_name: str
        :return: A :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticReadGroup` object.
        :rtype: FilePathTrimmomaticReadGroup
        """
        return FilePathTrimmomaticReadGroup(
            prefix=cls.get_prefix_read_group(read_group_name=read_group_name))

    @classmethod
    def get_file_path_project(cls, project_name: str, prefix_analysis: str) -> FilePathTrimmomaticProject:
        """Get a :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticProject` object.

        :param project_name: A project name.
        :type project_name: str
        :param prefix_analysis: A :py:attr:`bsf.analysis.Analysis.prefix` attribute.
        :type prefix_analysis: str
        :return: A :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticProject` object.
        :rtype: FilePathTrimmomaticProject
        """
        return FilePathTrimmomaticProject(
            prefix=cls.get_prefix_project(),
            prefix_analysis=prefix_analysis,
            project_name=project_name)

    def __init__(
            self,
            configuration: Optional[Configuration] = None,
            project_name: Optional[str] = None,
            genome_version: Optional[str] = None,
            input_directory: Optional[str] = None,
            output_directory: Optional[str] = None,
            project_directory: Optional[str] = None,
            genome_directory: Optional[str] = None,
            report_style_path: Optional[str] = None,
            report_header_path: Optional[str] = None,
            report_footer_path: Optional[str] = None,
            e_mail: Optional[str] = None,
            stage_list: Optional[list[Stage]] = None,
            collection: Optional[Collection] = None,
            sample_list: Optional[list[Sample]] = None,
            adapter_path: Optional[str] = None,
            trimming_step_pe_list: Optional[list[str]] = None,
            trimming_step_se_list: Optional[list[str]] = None,
            java_archive_trimmomatic: Optional[str] = None) -> None:
        """Initialise a :py:class:`bsf.analyses.trimmomatic.Trimmomatic` object.

        :param configuration: A :py:class:`bsf.standards.Configuration` object.
        :type configuration: Configuration | None
        :param project_name: A project name.
        :type project_name: str | None
        :param genome_version: A genome assembly version.
        :type genome_version: str | None
        :param input_directory: An input directory path.
        :type input_directory: str | None
        :param output_directory: An output directory path.
        :type output_directory: str | None
        :param project_directory: A project directory path, normally under the output directory path.
        :type project_directory: str | None
        :param genome_directory: A genome directory path, normally under the project directory path.
        :type genome_directory: str | None
        :param report_style_path: A report style :literal:`CSS` file path.
        :type report_style_path: str | None
        :param report_header_path: A report header :literal:`XHTML 1.0` file path.
        :type report_header_path: str | None
        :param report_footer_path: A report footer :literal:`XHTML 1.0` file path.
        :type report_footer_path: str | None
        :param e_mail: An e-mail address for a :emphasis:`UCSC Genome Browser Track Hub`.
        :type e_mail: str | None
        :param stage_list: A Python :py:class:`list` object of :py:class:`bsf.analysis.Stage` objects.
        :type stage_list: list[Stage] | None
        :param collection: A :py:class:`bsf.ngs.Collection` object.
        :type collection: Collection | None
        :param sample_list: A Python :py:class:`list` object of :py:class:`bsf.ngs.Sample` objects.
        :type sample_list: list[Sample] | None
        :param adapter_path: An adapter definition file path.
        :type adapter_path: str | None
        :param trimming_step_pe_list: A colon-separated list of Trimmomatic steps for paired-end data.
        :type trimming_step_pe_list: list[str] | None
        :param trimming_step_se_list: A colon-separated list of Trimmomatic steps for single-end data.
        :type trimming_step_se_list: list[str] | None
        :param java_archive_trimmomatic: A Trimmomatic tool :emphasis:`Java Archive` (JAR) file path.
        :type java_archive_trimmomatic: str | None
        """
        super(Trimmomatic, self).__init__(
            configuration=configuration,
            project_name=project_name,
            genome_version=genome_version,
            input_directory=input_directory,
            output_directory=output_directory,
            project_directory=project_directory,
            genome_directory=genome_directory,
            report_style_path=report_style_path,
            report_header_path=report_header_path,
            report_footer_path=report_footer_path,
            e_mail=e_mail,
            stage_list=stage_list,
            collection=collection,
            sample_list=sample_list)

        self.adapter_path = adapter_path
        self.trimming_step_pe_list = trimming_step_pe_list
        self.trimming_step_se_list = trimming_step_se_list
        self.java_archive_trimmomatic = java_archive_trimmomatic

        return

    def set_configuration(self, configuration: Configuration, section: str) -> None:
        """Set instance variables of a :py:class:`bsf.analyses.trimmomatic.Trimmomatic` object
        via a section of a :py:class:`bsf.standards.Configuration` object.

        Instance variables without a configuration option remain unchanged.

        :param configuration: A :py:class:`bsf.standards.Configuration` object.
        :type configuration: Configuration
        :param section: A configuration file section.
        :type section: str
        """
        super(Trimmomatic, self).set_configuration(configuration=configuration, section=section)

        # Sub-class specific ...

        option = 'adapter_path'
        if configuration.config_parser.has_option(section=section, option=option):
            self.adapter_path = configuration.config_parser.get(section=section, option=option)

        # Get the list of default trimming steps for paired-end data mode.

        option = 'trimming_steps_pe'
        if configuration.config_parser.has_option(section=section, option=option):
            self.trimming_step_pe_list = configuration.get_list_from_csv(section=section, option=option)

        # Get the list of default trimming steps for single-end data mode.

        option = 'trimming_steps_se'
        if configuration.config_parser.has_option(section=section, option=option):
            self.trimming_step_se_list = configuration.get_list_from_csv(section=section, option=option)

        # Get the Trimmomatic tool Java Archive (JAR) file path.

        option = 'java_archive_trimmomatic'
        if configuration.config_parser.has_option(section=section, option=option):
            self.java_archive_trimmomatic = configuration.config_parser.get(section=section, option=option)

        return

    def run(self) -> None:
        """Run a :py:class:`bsf.analyses.trimmomatic.Trimmomatic` object.

        This method changes the :py:class:`bsf.analyses.trimmomatic.Trimmomatic.collection` attribute
        (:py:class:`bsf.ngs.Collection`) to update with FASTQ file paths.
        """

        def run_adjust_illumina_clip_path(trimming_step_list: list[str]) -> None:
            """Private function to adjust the adapter :emphasis:`FASTA` file path of
            :literal:`ILLUMINACLIP` trimming steps.

            If the file path is not absolute, prepend the value of the adapter_path
            instance variable.

            :param trimming_step_list: A Python :py:class:`list` object of
                Python :py:class:`str` (trimming step) objects.
            :type trimming_step_list: list[str]
            """
            for i in range(0, len(trimming_step_list)):
                if trimming_step_list[i].startswith('ILLUMINACLIP'):
                    component_list = trimming_step_list[i].split(':')
                    if not os.path.isabs(component_list[1]):
                        component_list[1] = os.path.join(self.adapter_path, component_list[1])
                    trimming_step_list[i] = ':'.join(component_list)
            return

        def run_read_comparisons() -> None:
            """Private function to read a :py:class:`bsf.annotation.AnnotationSheet` specifying comparisons
            from a CSV file path.

            This implementation just adds all :py:class:`bsf.ngs.Sample` objects from the
            :py:attr:`bsf.analysis.Analysis.collection` instance variable
            (i.e., :py:class:`bsf.ngs.Collection` object) to the
            :py:attr:`bsf.analysis.Analysis.sample_list` instance variable.
            """

            self.sample_list.extend(self.collection.get_all_samples())

            return

        # Start of the run() method body.

        super(Trimmomatic, self).run()

        # Get the Trimmomatic tool Java Archive (JAR) file path.

        if not self.java_archive_trimmomatic:
            self.java_archive_trimmomatic = JavaArchive.get_trimmomatic()
            if not self.java_archive_trimmomatic:
                raise Exception(f"A {self.name!s} requires a 'java_archive_trimmomatic' configuration option.")

        if not (self.adapter_path and os.path.isabs(self.adapter_path)):
            self.adapter_path = os.path.join(os.path.dirname(self.java_archive_trimmomatic), 'adapters')

        if self.trimming_step_pe_list is None:
            raise Exception(f"A {self.name!s} requires a 'trimming_steps_pe' configuration option.")

        if self.trimming_step_se_list is None:
            raise Exception(f"A {self.name!s} requires a 'trimming_steps_se' configuration option.")

        run_adjust_illumina_clip_path(trimming_step_list=self.trimming_step_pe_list)
        run_adjust_illumina_clip_path(trimming_step_list=self.trimming_step_se_list)

        run_read_comparisons()

        # Trimmomatic

        stage_read_group = self.get_stage(name=self.get_stage_name_read_group())
        stage_summary = self.get_stage(name=self.get_stage_name_summary())
        stage_project = self.get_stage(name=self.get_stage_name_project())

        project_dependency_list: list[str] = list()

        # Sort the Python list of Sample objects by Sample.name.

        self.sample_list.sort(key=lambda item: item.name)

        for sample in self.sample_list:
            module_logger.debug('Sample name: %r', sample.name)
            module_logger.log(logging.DEBUG - 2, 'Sample: %r', sample)

            sample_step_list: list[str] = list()
            if 'Trimmomatic Steps' in sample.annotation_dict:
                for trimming_step in sample.annotation_dict['Trimmomatic Steps']:
                    sample_step_list.extend(self.configuration.list_from_csv(csv_string=trimming_step))
                run_adjust_illumina_clip_path(trimming_step_list=sample_step_list)

            # The Trimmomatic analysis does not obey excluded PairedReads objects,
            # more high-level analyses generally do.
            paired_reads_dict = sample.get_all_paired_reads(replicate_grouping=False, exclude=False)

            for paired_reads_name in sorted(paired_reads_dict):
                for paired_reads in paired_reads_dict[paired_reads_name]:
                    module_logger.debug('PairedReads name: %r', paired_reads.get_name())
                    module_logger.log(logging.DEBUG - 2, 'PairedReads: %r', paired_reads)

                    paired_reads_step_list = list()
                    if 'Trimmomatic Steps' in paired_reads.annotation_dict:
                        for trimming_step in paired_reads.annotation_dict['Trimmomatic Steps']:
                            paired_reads_step_list.extend(self.configuration.list_from_csv(csv_string=trimming_step))
                        run_adjust_illumina_clip_path(trimming_step_list=paired_reads_step_list)

                    # Apply some sanity checks.

                    # Maybe this case should be allowed after Trimmomatic trimming,
                    # where only the second Read survives.
                    if paired_reads.reads_1 is None and paired_reads.reads_2 is not None:
                        raise Exception(f"The PairedReads object {paired_reads.get_name()!r} has a 'reads_2', "
                                        f"but no reads_1 object.")

                    file_path_read_group = self.get_file_path_read_group(read_group_name=paired_reads_name)

                    # Create a Runnable and an Executable for running the Trimmomatic analysis.

                    runnable_read_group = self.add_runnable(
                        runnable=ConsecutiveRunnable(
                            name=self.get_prefix_read_group(read_group_name=paired_reads_name),
                            working_directory=self.project_directory))
                    self.set_stage_runnable(stage=stage_read_group, runnable=runnable_read_group)

                    # Record the Executable.name for the project dependency.

                    project_dependency_list.append(runnable_read_group.name)

                    # Create a new RunnableStepMakeDirectory in preparation of the Trimmomatic program.

                    runnable_step_read_group = RunnableStepMakeDirectory(
                        name='mkdir',
                        directory_path=file_path_read_group.output_directory)
                    runnable_read_group.add_runnable_step(runnable_step=runnable_step_read_group)

                    # Create a RunnableStep for the Trimmomatic program.

                    runnable_step_read_group = RunnableStepJava(
                        name='trimmomatic',
                        java_temporary_path=runnable_read_group.temporary_directory_path(absolute=False),
                        java_heap_maximum='Xmx4G',
                        java_jar_path=self.java_archive_trimmomatic)
                    runnable_read_group.add_runnable_step(runnable_step=runnable_step_read_group)

                    if paired_reads.reads_2 is None:
                        runnable_step_read_group.sub_command.sub_command = Command(program='SE')
                    else:
                        runnable_step_read_group.sub_command.sub_command = Command(program='PE')

                    # Add options to the sub command.
                    sub_command = runnable_step_read_group.sub_command.sub_command
                    sub_command.add_option_short(key='trimlog', value=file_path_read_group.trim_log_tsv)

                    if paired_reads.reads_2 is None:
                        file_path_read_group.reads_1u = os.path.join(
                            file_path_read_group.output_directory,
                            paired_reads.reads_1.name + 'U.fastq.gz')

                        sub_command.arguments.append(paired_reads.reads_1.file_path)
                        sub_command.arguments.append(file_path_read_group.reads_1u)

                        # Update unpaired Reads information.

                        paired_reads.reads_1.name += 'U'
                        paired_reads.reads_1.file_path = os.path.join(
                            self.genome_directory,
                            file_path_read_group.reads_1u)
                    else:
                        file_path_read_group.reads_1p = os.path.join(
                            file_path_read_group.output_directory,
                            paired_reads.reads_1.name + 'P.fastq.gz')
                        file_path_read_group.reads_1u = os.path.join(
                            file_path_read_group.output_directory,
                            paired_reads.reads_1.name + 'U.fastq.gz')
                        file_path_read_group.reads_2p = os.path.join(
                            file_path_read_group.output_directory,
                            paired_reads.reads_2.name + 'P.fastq.gz')
                        file_path_read_group.reads_2u = os.path.join(
                            file_path_read_group.output_directory,
                            paired_reads.reads_2.name + 'U.fastq.gz')

                        sub_command.arguments.append(paired_reads.reads_1.file_path)
                        sub_command.arguments.append(paired_reads.reads_2.file_path)
                        sub_command.arguments.append(file_path_read_group.reads_1p)
                        sub_command.arguments.append(file_path_read_group.reads_1u)
                        sub_command.arguments.append(file_path_read_group.reads_2p)
                        sub_command.arguments.append(file_path_read_group.reads_2u)

                        # Update paired Reads information.

                        paired_reads.reads_1.name += 'P'
                        paired_reads.reads_1.file_path = os.path.join(
                            self.genome_directory,
                            file_path_read_group.reads_1p)
                        paired_reads.reads_2.name += 'P'
                        paired_reads.reads_2.file_path = os.path.join(
                            self.genome_directory,
                            file_path_read_group.reads_2p)

                        # Add unpaired Reads 1 and 2 as separate PairedReads objects to this sample.

                        sample.add_paired_reads(
                            paired_reads=PairedReads(
                                annotation_dict=paired_reads.annotation_dict,
                                reads_1=Reads(
                                    name=paired_reads.reads_1.name[:-1] + 'U',
                                    file_path=os.path.join(
                                        self.genome_directory,
                                        file_path_read_group.reads_1u)),
                                exclude=paired_reads.exclude,
                                index_1=paired_reads.index_1,
                                index_2=paired_reads.index_2,
                                read_group=paired_reads.read_group))

                        sample.add_paired_reads(
                            paired_reads=PairedReads(
                                annotation_dict=paired_reads.annotation_dict,
                                reads_1=Reads(
                                    name=paired_reads.reads_2.name[:-1] + 'U',
                                    file_path=os.path.join(
                                        self.genome_directory,
                                        file_path_read_group.reads_2u)),
                                exclude=paired_reads.exclude,
                                index_1=paired_reads.index_1,
                                index_2=paired_reads.index_2,
                                read_group=paired_reads.read_group))

                    # Append trimming steps in order of specificity read group, sample and default.

                    if len(paired_reads_step_list):
                        sub_command.arguments.extend(paired_reads_step_list)
                    elif len(sample_step_list):
                        sub_command.arguments.extend(sample_step_list)
                    else:
                        if paired_reads.reads_2 is None:
                            sub_command.arguments.extend(self.trimming_step_se_list)
                        else:
                            sub_command.arguments.extend(self.trimming_step_pe_list)

                    # Create a new RunnableStep to compress the trim log file.

                    runnable_step_read_group = RunnableStep(
                        name='compress_logs',
                        program='pigz')
                    runnable_read_group.add_runnable_step(runnable_step=runnable_step_read_group)

                    runnable_step_read_group.add_switch_long(key='best')
                    runnable_step_read_group.add_option_long(key='processes', value=str(stage_summary.threads))
                    runnable_step_read_group.arguments.append(file_path_read_group.trim_log_tsv)

                    # Create a Runnable for the bsf_trimmomatic_summary.R analysis.

                    prefix_summary = '_'.join((stage_summary.name, paired_reads_name))

                    runnable_summary = self.add_runnable(
                        runnable=ConsecutiveRunnable(
                            name=prefix_summary,
                            working_directory=self.project_directory))
                    executable_summary = self.set_stage_runnable(stage=stage_summary, runnable=runnable_summary)
                    executable_summary.dependencies.append(runnable_read_group.name)

                    # Create a new RunnableStep to aggregate the trim log file.

                    runnable_step_summary = RunnableStep(
                        name='trimmomatic_summary',
                        program='bsf_trimmomatic_summary.R',
                        obsolete_file_path_list=[
                            # file_path_read_group.trim_log_tsv_gz,
                        ])
                    runnable_summary.add_runnable_step(runnable_step=runnable_step_summary)

                    runnable_step_summary.add_option_long(
                        key='file-path',
                        value=file_path_read_group.trim_log_tsv_gz)

        # Create a Runnable for pruning the sample annotation sheet.

        prefix_project = '_'.join((stage_project.name, self.project_name))

        file_path_project = self.get_file_path_project(project_name=self.project_name, prefix_analysis=self.prefix)

        # Convert the (modified) Collection object into a SampleAnnotationSheet object and write it to disk.

        annotation_sheet = self.collection.to_sas(
            file_path=os.path.join(self.project_directory, file_path_project.sas_path_old),
            name=prefix_project)
        annotation_sheet.to_file_path()

        runnable_project = self.add_runnable(
            runnable=ConsecutiveRunnable(
                name=prefix_project,
                working_directory=self.project_directory))
        executable_project = self.set_stage_runnable(
            stage=stage_project,
            runnable=runnable_project)
        executable_project.dependencies.extend(project_dependency_list)

        # Create a new RunnableStep.

        runnable_step_project = RunnableStepCollectionPruneFastq(
            name='prune_sample_annotation_sheet',
            obsolete_file_path_list=[
                # file_path_project.sas_path_old,
            ],
            file_path_old=file_path_project.sas_path_old,
            file_path_new=file_path_project.sas_path_new,
            minimum_size=1024)
        runnable_project.add_runnable_step(runnable_step=runnable_step_project)

        return

    def report(self) -> None:
        """Create a :literal:`XHTML 1.0` report and a :emphasis:`UCSC Genome Browser Track Hub`.
        """

        # Create a symbolic link containing the project name and a UUID.
        self.create_public_project_link()

        # Write a HTML document.

        report_list: list[str] = list()

        report_list.append('<h1 id="' + self.prefix + '_analysis">' + self.project_name + ' ' + self.name + '</h1>\n')
        report_list.append('\n')

        report_list.append('<h2 id="aliquot_and_sample_level">Aliquot and Sample Level</h2>\n')
        report_list.append('\n')
        report_list.append('<table id="aliquot_and_sample_table">\n')
        report_list.append('<thead>\n')
        report_list.append('<tr>\n')
        report_list.append('<th>Sample</th>\n')
        report_list.append('<th>Aliquot</th>\n')
        report_list.append('<th>Coverage</th>\n')
        report_list.append('<th>Frequency</th>\n')
        report_list.append('<th>Summary</th>\n')
        report_list.append('</tr>\n')
        report_list.append('</thead>\n')
        report_list.append('<tbody>\n')

        for sample in self.sample_list:
            # The Trimmomatic analysis does not obey excluded PairedReads objects,
            # more high-level analyses generally do.
            paired_reads_dict = sample.get_all_paired_reads(replicate_grouping=False, exclude=False)

            if not paired_reads_dict:
                # Skip Sample objects, which PairedReads objects have all been excluded.
                continue

            report_list.append('<tr>\n')
            report_list.append('<td class="left">' + sample.name + '</td>\n')
            report_list.append('<td class="left"></td>\n')  # Aliquot
            report_list.append('<td class="center"></td>\n')  # Coverage PNG
            report_list.append('<td class="center"></td>\n')  # Frequency PNG
            report_list.append('<td class="center"></td>\n')  # Summary TSV
            report_list.append('</tr>\n')

            # This analysis is special in that read group names carry 'P' or 'U' suffices and samples carry additional
            # read groups after trimming that do no longer correspond to the initial Runnable objects. Sigh.
            # Transiently create a Python dict without the suffix and sort its keys (bsf.ngs.PairedReads.name).

            for paired_reads_name in sorted(dict(map(lambda item: (item[:-1], True), paired_reads_dict.keys()))):
                prefix_read_group = self.get_prefix_read_group(read_group_name=paired_reads_name)

                # The second read may still not be there.
                if prefix_read_group not in self.runnable_dict:
                    continue

                runnable_read_group = self.runnable_dict[prefix_read_group]
                file_path_read_group = self.get_file_path_read_group(read_group_name=paired_reads_name)

                report_list.append('<tr>\n')
                # Sample
                report_list.append('<td class="left"></td>\n')
                # Aliquot
                report_list.append('<td class="left">' + paired_reads_name + '</td>\n')
                # Coverage
                report_list.append('<td class="center">')
                report_list.append('<a href="' + file_path_read_group.coverage_png + '">')
                report_list.append('<img alt="Coverage ' + runnable_read_group.name + '"')
                report_list.append(' src="' + file_path_read_group.coverage_png + '"')
                report_list.append(' height="100" width="100" />')
                report_list.append('</a>')
                report_list.append('</td>\n')
                # Frequency
                report_list.append('<td class="center">')
                report_list.append('<a href="' + file_path_read_group.frequency_png + '">PNG</a>')
                report_list.append('</td>\n')
                # The frequency plots provide little information that does not necessarily justify
                # adding another set of images onto the HTML report.
                report_list.append('<td class="center">')
                report_list.append('<a href="' + file_path_read_group.summary_tsv + '">TSV</a>')
                report_list.append('</td>\n')
                report_list.append('</tr>\n')

        report_list.append('</tbody>\n')
        report_list.append('</table>\n')
        report_list.append('\n')

        self.report_to_file(content=report_list)

        return

    @classmethod
    def console_submit(
            cls,
            configuration_path: str,
            stage_name: Optional[str] = None,
            drms_submit: Optional[bool] = None,
            project_name: Optional[str] = None,
            sas_path: Optional[str] = None,
            *args, **kwargs) -> int:
        """Console function to submit a :py:class:`bsf.analyses.trimmomatic.Trimmomatic` analysis.

        This analysis requires either a :literal:`configuration` argument or
        :literal:`project_name` and :literal:`sas_path` arguments.

        :param configuration_path: A configuration `INI <https://en.wikipedia.org/wiki/INI_file>`_ file path.
        :type configuration_path: str
        :param stage_name: A :py:class:`bsf.analysis.Stage` name.
        :type stage_name: str | None
        :param drms_submit: Request submitting into the DRMS.
        :type drms_submit: bool | None
        :param project_name: A project name.
        :type project_name: str
        :param sas_path: A sample annotation sheet file path.
        :type sas_path: str
        :return: A :py:class:`SystemExit` status value.
        :rtype: int
        """
        if configuration_path == Configuration.get_global_file_path():
            if not project_name:
                raise Exception('The --project-name argument is required if configuration is not set.')
            if not sas_path:
                raise Exception('The --sas-path argument is required if configuration is not set.')

        analysis: Trimmomatic = cls.from_config_file_path(config_path=configuration_path)

        if project_name:
            if project_name.endswith('.ini'):
                raise Exception('The --project-name option should not be a configuration (INI) file.')

            analysis.project_name = project_name

        if sas_path:
            if not sas_path.endswith('.csv'):
                raise Exception('The --sas-path option requires a comma-separated value (CSV) file.')

            analysis.sas_file = sas_path

        analysis.run()
        analysis.check_state()
        analysis.submit(name=stage_name, drms_submit=drms_submit)

        print(analysis.name)
        print('Project name:      ', analysis.project_name)
        print('Input directory:   ', analysis.input_directory)
        print('Project directory: ', analysis.project_directory)
        print('Genome directory:  ', analysis.genome_directory)
        if stage_name == 'report':
            print('Report URL:        ', analysis.get_html_report_url())

        return 0

    @classmethod
    def entry_point_submit(cls) -> int:
        """Console entry point to submit a :py:class:`bsf.analyses.trimmomatic.Trimmomatic` analysis.

        This analysis requires either a positional :literal:`configuration` argument or
        :literal:`--project-name` and :literal:`--sas-path` arguments.

        :return: A :py:class:`SystemExit` status value.
        :rtype: int
        """
        argument_parser = ArgumentParser(
            description=cls.name + ' submission script.')

        argument_parser.add_argument(
            '--dry-run',
            action='store_false',
            help='dry run',
            dest='drms_submit')

        argument_parser.add_argument(
            '--logging-level',
            default='WARNING',
            choices=('CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'DEBUG1', 'DEBUG2'),
            help='logging level [WARNING]')

        argument_parser.add_argument(
            '--stage-name',
            help='limit job submission to a particular analysis stage')

        argument_parser.add_argument(
            '--project-name',
            help='project name')

        argument_parser.add_argument(
            '--sas-path',
            help='sample annotation sheet (CSV) file path')

        argument_parser.add_argument(
            'configuration',
            nargs='?',
            default=Configuration.get_global_file_path(),
            help=f'configuration (INI) file path [{Configuration.get_global_file_path()!s}]')

        name_space = argument_parser.parse_args()

        if name_space.logging_level:
            logging.addLevelName(level=logging.DEBUG - 1, levelName='DEBUG1')
            logging.addLevelName(level=logging.DEBUG - 2, levelName='DEBUG2')

            logging.basicConfig(level=name_space.logging_level)

        return cls.console_submit(
            configuration_path=name_space.configuration,
            stage_name=name_space.stage_name,
            drms_submit=name_space.drms_submit,
            project_name=name_space.project_name,
            sas_path=name_space.sas_path)


from DataVisualization import DataVisualization
from Header import Header
from Navigation import Navigation
from PetrophysicsTopBar import PetrophysicsTopBar
from PetrophysicsOptions import PetrophysicsOptions
from PlotViewer import PlotViewer
from Spreadsheet import Spreadsheet, SpreadsheetControls
from StatisticsOptions import StatisticsOptions
from StatisticsTopBar import StatisticsTopBar
from PySide2.QtWidgets import QMainWindow, QStackedLayout, QWidget, QDockWidget, QVBoxLayout, QApplication, QSizePolicy, \
    QScrollArea, QFileDialog, QMessageBox, QLabel, QStatusBar
from PySide2.QtCore import Qt, Signal
from PySide2.QtGui import QIcon
import Data
import sys
from HandlerThread import HandlerThread
from Regressions import regressionManifold
from Statistics import statisticsManifold
from Boxplot import boxplotManifold
from Histogram import histogramManifold
from DockWidgetRibbon import DockWidgetRibbon
from KozenyCarman import kCarmanManifold
from Lucia import luciaManifold
from FlowZones import fzManifold
from Winland import winlandManifold
from DykstraParsons import dParsonsManifold
from TimurCoates import tCoatesManifold
from PlotControls import PlotControls
from PlotControleEngine import pceManifold

class MainWindow(QMainWindow):
    changePage = Signal(bool)
    def __init__(self):
        super().__init__()
        self.setWindowTitle('Petro-Explorer v1.0')
        win_icon = QIcon('icons/Logo.ico')
        self.setWindowIcon(win_icon)
        self.setStyleSheet('background-color: #363f49;')

        self.header = Header()
        self.navigation = Navigation()
        self.p_top_bar = PetrophysicsTopBar()
        self.s_top_bar = StatisticsTopBar()
        self.p_options_1 = PetrophysicsOptions(_mode='KCarman')
        self.p_options_2 = PetrophysicsOptions(_mode='TCoates')
        self.p_options_3 = PetrophysicsOptions(_mode='Winland')
        self.p_options_4 = PetrophysicsOptions(_mode='RQIFZI')
        self.p_options_5 = PetrophysicsOptions(_mode='Lucia')
        self.p_options_6 = PetrophysicsOptions(_mode='DParsons')
        self.s_options_1 = StatisticsOptions(_mode='Regression')
        self.s_options_2 = StatisticsOptions(_mode='Statistics')
        self.s_options_3 = StatisticsOptions(_mode='Histogram')
        self.s_options_4 = StatisticsOptions(_mode='Boxplot')
        self.sp_controls = SpreadsheetControls()
        self.plot_controls = PlotControls()
        self.plot_viewer = PlotViewer()
        self.spreadsheet = Spreadsheet()
        self.status_bar = QStatusBar()

        self.scroll_area_1 = QScrollArea()
        self.scroll_area_1.setWidget(self.spreadsheet)
        self.scroll_area_1.setWidgetResizable(True)

        self.scroll_area_2 = QScrollArea()
        self.scroll_area_2.setWidget(self.plot_viewer)
        self.scroll_area_2.setWidgetResizable(True)

        self.bar_widget = QWidget()
        self.bar_widget.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Maximum)

        self.central_widget = QWidget()
        self.options_widget = QWidget()

        self.docking_options = QDockWidget()
        self.docking_options.setWidget(self.options_widget)
        self.docking_options.setTitleBarWidget(DockWidgetRibbon(' Opções de cálculo'))

        self.docking_options2 = QDockWidget()
        self.docking_options2.setWidget(self.sp_controls)
        self.docking_options2.setTitleBarWidget(DockWidgetRibbon(' Controles de visualização'))

        self.setCentralWidget(self.central_widget)
        self.setStatusBar(self.status_bar)
        self.addToolBar(self.navigation)
        self.addDockWidget(Qt.DockWidgetArea.RightDockWidgetArea, self.docking_options)
        self.addDockWidget(Qt.DockWidgetArea.RightDockWidgetArea, self.docking_options2)
        self.connections()
        self.buildLayout()

        self.centralWidget().setStyleSheet('background-color: #2e3843')
        self.spreadsheet.setStyleSheet('background-color: white')
        self.status_bar.setStyleSheet('color: white')

    def buildLayout(self):
        stacked_layout_1 = QStackedLayout()
        # stacked_layout_1.addWidget(QWidget())
        stacked_layout_1.addWidget(self.p_top_bar)
        stacked_layout_1.addWidget(self.s_top_bar)

        stacked_layout_2 = QStackedLayout()
        # stacked_layout_2.addWidget(QWidget())
        stacked_layout_2.addWidget(self.p_options_1)
        stacked_layout_2.addWidget(self.p_options_2)
        stacked_layout_2.addWidget(self.p_options_3)
        stacked_layout_2.addWidget(self.p_options_4)
        stacked_layout_2.addWidget(self.p_options_5)
        stacked_layout_2.addWidget(self.p_options_6)
        stacked_layout_2.addWidget(self.s_options_1)
        stacked_layout_2.addWidget(self.s_options_2)
        stacked_layout_2.addWidget(self.s_options_3)
        stacked_layout_2.addWidget(self.s_options_4)

        self.stacked_layout_3 = QStackedLayout()
        self.stacked_layout_3.addWidget(self.scroll_area_1)
        self.stacked_layout_3.addWidget(self.scroll_area_2)

        central_widget_layout = QVBoxLayout()
        central_widget_layout.addWidget(self.bar_widget)
        central_widget_layout.addLayout(self.stacked_layout_3)

        self.central_widget.setLayout(central_widget_layout)
        self.bar_widget.setLayout(stacked_layout_1)
        self.options_widget.setLayout(stacked_layout_2)

    def connections(self):
        self.navigation.PetroAnalysis.connect(lambda: self.bar_widget.layout().setCurrentIndex(0))
        self.navigation.StatsAnalysis.connect(lambda: self.bar_widget.layout().setCurrentIndex(1))
        self.navigation.Save.connect(lambda: self.saveDialog())
        self.navigation.Import.connect(lambda: self.importDialog())
        self.navigation.About.connect(lambda: self.aboutDialog())
        self.navigation.Help.connect(lambda: self.helpDialog())
        self.navigation.header.HomePage.connect(lambda: self.changePage.emit(True))
        self.navigation.ViewSheet.connect(lambda: self.displaySheet())
        self.navigation.ViewPlot.connect(lambda: self.displayPltE())
        self.navigation.New.connect(lambda: self.spreadsheet.addBlankSheet())

        self.p_top_bar.KCarman.connect(lambda: self.options_widget.layout().setCurrentIndex(0))
        self.p_top_bar.TCoates.connect(lambda: self.options_widget.layout().setCurrentIndex(1))
        self.p_top_bar.Winland.connect(lambda: self.options_widget.layout().setCurrentIndex(2))
        self.p_top_bar.DParsons.connect(lambda: self.options_widget.layout().setCurrentIndex(5))
        self.p_top_bar.Lucia.connect(lambda: self.options_widget.layout().setCurrentIndex(4))
        self.p_top_bar.RF.connect(lambda: self.options_widget.layout().setCurrentIndex(3))

        self.s_top_bar.Regr.connect(lambda: self.options_widget.layout().setCurrentIndex(6))
        self.s_top_bar.StatDesc.connect(lambda: self.options_widget.layout().setCurrentIndex(7))
        self.s_top_bar.Boxplt.connect(lambda: self.options_widget.layout().setCurrentIndex(9))
        self.s_top_bar.Hist.connect(lambda: self.options_widget.layout().setCurrentIndex(8))

        self.s_options_1.run_button.clicked.connect(self.startRegr)
        self.s_options_2.run_button.clicked.connect(self.startStat)
        self.s_options_3.run_button.clicked.connect(self.startHist)
        self.s_options_4.run_button.clicked.connect(self.startBxpl)

        self.p_options_1.run.clicked.connect(self.startKCoz)
        self.p_options_2.run.clicked.connect(self.startTCoa)
        self.p_options_3.run.clicked.connect(self.startWinl)
        self.p_options_4.run.clicked.connect(self.startFZIR)
        self.p_options_5.run.clicked.connect(self.startLuci)
        self.p_options_6.run.clicked.connect(self.startDyks)

        self.sp_controls.AddColumn.connect(lambda: self.spreadsheet.addColumn())
        self.sp_controls.AddRow.connect(lambda: self.spreadsheet.addRow())
        self.sp_controls.DeleteRow.connect(lambda: self.spreadsheet.deleteRow())
        self.sp_controls.DeleteColumn.connect(lambda: self.spreadsheet.deleteColumn())

        self.plot_controls.run_button.clicked.connect(lambda: self.startPltE())
        self.plot_controls.erasePlot.connect(lambda: self.plot_viewer.erasePlot())

        self.plot_viewer.feedPlotControls.connect(self.plot_controls.fillFromJson)

    def startPltE(self):
        old_json = self.plot_viewer.json_data
        title = self.plot_controls.modify_title.text()
        xtype = self.plot_controls.modify_x_axis_type.currentText()
        ytype = self.plot_controls.modify_y_axis_type.currentText()
        xlabel = self.plot_controls.modify_x_axis_label.text()
        ylabel = self.plot_controls.modify_y_axis_label.text()

        if len(self.plot_controls.modify_lower_x_range.text()) > 0:
            lxrange = float(self.plot_controls.modify_lower_x_range.text())
        else:
            lxrange = None
        if len(self.plot_controls.modify_upper_x_range.text()) > 0:
            uxrange = float(self.plot_controls.modify_upper_x_range.text())
        else:
            uxrange = None
        if len(self.plot_controls.modify_lower_y_range.text()) > 0:
            lyrange = float(self.plot_controls.modify_lower_y_range.text())
        else:
            lyrange = None
        if len(self.plot_controls.modify_upper_y_range.text()) > 0:
            uyrange = float(self.plot_controls.modify_upper_y_range.text())
        else:
            uyrange = None

        if len(self.plot_controls.add_x_trace.text()) > 0 and len(self.plot_controls.add_y_trace.text()):
            trace_type = self.plot_controls.type_of_trace.currentText()
            x_trace = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.plot_controls.add_x_trace.text())]
            y_trace = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.plot_controls.add_y_trace.text())]
            trace_name = self.plot_controls.trace_name.text()
        else:
            trace_type = None
            x_trace = None
            y_trace = None
            trace_name = None

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.loadPltE(io_operations_handler.results))
        io_operations_handler.loadParameters(f=pceManifold, _args=[old_json, title, xtype, ytype, xlabel, ylabel, lxrange, uxrange, lyrange, uyrange, trace_type, x_trace, y_trace, trace_name])
        io_operations_handler.start()

    def loadPltE(self, results, typ='Regressão'):
        self.plot_viewer.loadPlot(results[0], results[1], _type=typ)
        self.displayPltE()

    def displayPltE(self):
        self.stacked_layout_3.setCurrentIndex(1)
        self.docking_options2.setWidget(self.plot_controls)
        #self.removeDockWidget(self.docking_options3)
        #self.addDockWidget(Qt.DockWidgetArea.RightDockWidgetArea, self.docking_options2)

    def loadSheet(self, df):
        self.spreadsheet.changeModel(data=df, header=list(df.keys()))
        self.displaySheet()

    def displaySheet(self):
        self.stacked_layout_3.setCurrentIndex(0)
        self.docking_options2.setWidget(self.sp_controls)
        self.status_bar.clearMessage()
        #self.removeDockWidget(self.docking_options2)
        #self.addDockWidget(Qt.DockWidgetArea.RightDockWidgetArea, self.docking_options3)

    def importDialog(self):
        func = None
        file_settings = QFileDialog().getOpenFileName(self, 'Importar Arquivo',
                                                      filter="Todos os arquivos (*);; Arquivo de Texto (*.txt);; Arquivo CSV (*.csv);; "
                                                             "Planilha Excel (*.xlsx)")
        if ".txt" in file_settings[0]:
            func = Data.readTXT
        if ".csv" in file_settings[0]:
            func = Data.readCSV
        if ".xlsx" in file_settings[0]:
            func = Data.readExcel

        self.status_bar.showMessage('Importando arquivo. Aguarde.')
        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.loadSheet(io_operations_handler.results))
        io_operations_handler.loadParameters(f=func, _args=[file_settings[0],])
        io_operations_handler.start()

    def saveDialog(self):
        func = None
        file_settings = QFileDialog().getSaveFileName(self, "Salvar Arquivo",
                                                      filter="Arquivo de Texto (*.txt);; Arquivo CSV (*.csv);; "
                                                             "Planilha Excel (*.xlsx)")
        if ".txt" in file_settings[0]:
            func = Data.toTXT
        if ".csv" in file_settings[0]:
            func = Data.toCSV
        if ".xlsx" in file_settings[0]:
            func = Data.toExcel

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.loadParameters(f=func, _args=[self.spreadsheet.retrieveModel(), file_settings[0]])
        io_operations_handler.start()

    def aboutDialog(self):
        icon = QIcon(r'icons/sobre.png')
        text = "O Petro-Explorer é um software desenvolvido no Laboratório de Exploração e Produção de Petróleo (" \
               "LENEP) da UENF para análises petrofísicas e estatísticas de dados de rocha. "
        msg = QMessageBox()
        msg.setWindowTitle('Sobre Petro-Explorer')
        msg.setWindowIcon(icon)
        msg.setText(text)
        msg.exec_()

    def helpDialog(self):
        icon = QIcon(r'icons/ajuda.png')
        text = r"Para utilizar o Petro-Explorer de maneira correta siga os seguintes passos:" \
               "\n1 - Clique no botão Novo na barra de navegação, isto irá limpar quaisquer dados de antigas análises.\n" \
               "2 - Para começar a sua análise é preciso importar os dados, assim, clique no botão Importar para escolher o seu arquivo. Atente para o fato que somente três tipos de arquivo são suportados (i.e. *.txt, *.csv, *.xlsx). Depois da sua escolha, os dados devem aparecer na planilha do software.\n" \
               "3 - Se você desejar realizar uma análise petrofísica, clique no botão de Análise Petrofísica na barra de navegação. Caso queira realizar uma análise estatística, clique no botão de Análise Estatística na barra de navegação.\n" \
               "4 - Selecione na barra superior à planilha, a análise que desejas realizar sobre seus dados.\n" \
               "5 - No canto direito da janela, selecione os parâmetros de sua análise assim como o destino de seus resultados. Clique no botão 'Começar Análise' para continuar.\n" \
               "6 - Analise seus resultados na planilha, e, quando disponível, no gráfico também.\n" \
               "7 - Quando terminar, clique no botão Salvar para salvar os resultados de suas análises no disco.\n" \
               "8 - Caso queira realizar outra análise, clique no botão \"Novo\" e comece novamente.\n"
        msg = QMessageBox()
        msg.setWindowTitle('Ajuda')
        msg.setWindowIcon(icon)
        msg.setText(text)
        msg.exec_()

    def startRegr(self):
        xdata = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.s_options_1.payload['x_column'])]
        ydata = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.s_options_1.payload['y_column'])]
        _type = self.s_options_1.payload['calculate']
        dgr = self.s_options_1.degree.value()

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.displayRegr(self.s_options_1.payload['output_column'], io_operations_handler.results))
        io_operations_handler.loadParameters(f=regressionManifold, _args=[xdata, ydata, dgr, _type])
        io_operations_handler.start()

    def startStat(self):
        xdata = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.s_options_2.payload['x_column'])]
        mthd = self.s_options_2.payload['calculate']
        qtl = self.s_options_2.quartile.value()
        pctl = self.s_options_2.percentile.value()

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.displayStat(self.s_options_2.payload['output_column'], io_operations_handler.results))
        io_operations_handler.loadParameters(f=statisticsManifold, _args=[xdata, mthd, qtl, pctl])
        io_operations_handler.start()

    def startHist(self):
        xdata = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.s_options_3.payload['x_column'])]
        nbins = self.s_options_3.payload['y_column']

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.displayHist(io_operations_handler.results))
        io_operations_handler.loadParameters(f=histogramManifold, _args=[xdata, nbins])
        io_operations_handler.start()

    def startBxpl(self):
        indexes = []
        columns = self.s_options_4.payload['column_range']
        for col in columns:
            indexes.append(self.spreadsheet.model.header_info.index(col))
        xdata = self.spreadsheet.model.input_data[:, indexes]
        box_orientation = self.s_options_4.payload['y_column']

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.displayBxpl(io_operations_handler.results))
        io_operations_handler.loadParameters(f=boxplotManifold, _args=[xdata, box_orientation])
        io_operations_handler.start()

    def startKCoz(self):
        #you need to ascertain if x corresponds to Permeability, y to Porosity and z to Swir/Svgr

        k = None
        phi = None
        svgr = None

        prp = self.p_options_1.payload['calculate']

        if prp == 'Permeabilidade (mD)':
            phi = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.p_options_1.payload['x_column'])]
            svgr = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.p_options_1.payload['y_column'])]
        elif prp == 'SVgr (cm-1)':
            k = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.p_options_1.payload['x_column'])]
            phi = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.p_options_1.payload['y_column'])]
        else:
            k = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.p_options_1.payload['x_column'])]
            phi = self.spreadsheet.model.input_data[:, self.spreadsheet.model.header_info.index(self.p_options_1.payload['y_column'])]

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.displayKCoz(io_operations_handler.results))
        io_operations_handler.loadParameters(f=kCarmanManifold, _args=[k, phi, svgr, prp])
        io_operations_handler.start()

    def startTCoa(self):
        #you need to ascertain if x corresponds to Permeability, y to Porosity and z to Swir/Svgr
        prp = self.p_options_2.payload['calculate']

        if prp == "Swir (%)":
            xdata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_2.payload['x_column'])]
            ydata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_2.payload['y_column'])]
            zdata = None
        else:
            xdata = None
            ydata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_2.payload['y_column'])]
            zdata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_2.payload['x_column'])]

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.displayTCoa(io_operations_handler.results))
        io_operations_handler.loadParameters(f=tCoatesManifold, _args=[xdata, ydata, zdata, prp])
        io_operations_handler.start()

    def startWinl(self):
        #you need to ascertain if x corresponds to Permeability, y to Porosity and z to Swir/Svgr
        if self.p_options_3.payload['x_column'] != '':
            xdata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_3.payload['x_column'])]
        else:
            xdata = []
        if self.p_options_3.payload['y_column'] != '':
            ydata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_3.payload['y_column'])]
        else:
            ydata = []

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.displayWinl(io_operations_handler.results))
        io_operations_handler.loadParameters(f=winlandManifold, _args=[xdata, ydata])
        io_operations_handler.start()

    def startFZIR(self):
        # you need to ascertain if x corresponds to Permeability, y to Porosity and z to Swir/Svgr
        if self.p_options_4.payload['x_column'] != '':
            xdata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_4.payload['x_column'])]
        else:
            xdata = None
        if self.p_options_4.payload['y_column'] != '':
            ydata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_4.payload['y_column'])]
        else:
            ydata = None
        if self.p_options_4.payload['z_column'] != '':
            zdata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_4.payload['z_column'])]
        else:
            zdata = None

        prp = self.p_options_4.payload['calculate']
        un = self.p_options_4.payload['column_range']

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.displayFZIR(io_operations_handler.results))
        io_operations_handler.loadParameters(f=fzManifold, _args=[xdata, ydata, zdata, un, prp])
        io_operations_handler.start()

    def startLuci(self):
        #you need to ascertain if x corresponds to Permeability, y to Porosity and z to Swir/Svgr
        if self.p_options_5.payload['x_column'] != '':
            xdata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_5.payload['x_column'])]
        else:
            xdata = []
        if self.p_options_5.payload['y_column'] != '':
            ydata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_5.payload['y_column'])]
        else:
            ydata = []

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.displayLuci(io_operations_handler.results))
        io_operations_handler.loadParameters(f=luciaManifold, _args=[xdata, ydata])
        io_operations_handler.start()

    def startDyks(self):
        #you need to ascertain if x corresponds to Permeability, y to Porosity and z to Swir/Svgr
        if self.p_options_6.payload['x_column'] != '':
            xdata = self.spreadsheet.model.input_data[:,
                    self.spreadsheet.model.header_info.index(self.p_options_6.payload['x_column'])]
        else:
            xdata = []
        prp = self.p_options_6.payload['calculate']

        io_operations_handler = HandlerThread()
        io_operations_handler.messageSent.connect(self.status_bar.showMessage)
        io_operations_handler.daemon = True
        io_operations_handler.hasFinished.connect(lambda: self.displayDyks(io_operations_handler.results))
        io_operations_handler.loadParameters(f=dParsonsManifold, _args=[xdata, prp])
        io_operations_handler.start()

    def displayKCoz(self, content):
        content = list(content)
        content.insert(0, 'Resultados - Kozeny-Carman')
        self.spreadsheet.addColumn(content)

    def displayWinl(self, content):
        self.loadPltE(content[:2])
        content[2].insert(0, 'R35 - Winland')
        content[3].insert(0, 'Ports - Winland')
        self.spreadsheet.addColumn(content[2])
        self.spreadsheet.addColumn(content[3])

    def displayTCoa(self, content):
        content.insert(0, 'Resultados - Timur Coates')
        self.spreadsheet.addColumn(content)

    def displayDyks(self, content):
        content.insert(0, 'Resultados - Dykstra-Parsons')
        self.spreadsheet.addColumn(content)
        self.p_options_6.results.setText('Atenção! Resultado pode ser diferente do coeficiente calculado através do gráfico de probabilidade. \n' + str(content[0]) + ': ' + str(content[1]))

    def displayLuci(self, results):
        self.loadPltE(results[:2])
        results[2].insert(0, 'Resultados - RFN/Lucia')
        self.spreadsheet.addColumn(results[2])

    def displayFZIR(self, results):
        results[0].insert(0, 'Resultados - RQI (μm)')
        self.spreadsheet.addColumn(results[0])
        results[1].insert(0, 'Resultados - PhiZ')
        self.spreadsheet.addColumn(results[1])
        self.loadPltE(results[2:])

    def displayRegr(self, display_name, results):
        self.s_options_1.results.setText(str(results[0]))
        self.loadPltE(results[-2:])

    def displayStat(self, display_name, results):
        self.s_options_2.results.setText(str(results))

    def displayHist(self, results):
        self.loadPltE(results, 'Histograma')

    def displayBxpl(self, results):
        self.loadPltE(results, 'Boxplot')

def test():
    app = QApplication(sys.argv)
    app.setWindowIcon(QIcon('icons/Logo.ico'))
    app.setStyle('Fusion')
    win = MainWindow()
    win.showMaximized()
    win.setWindowIcon(QIcon('icons/Logo.ico'))
    sys.exit(app.exec_())

if __name__ == "__main__":
    test()


#import sys
# insert at 1, 0 is the script path (or '' in REPL)
#sys.path.insert(1, 'C:/Users/gamer/source/repos/MMuhlenkort/PythonWalser/ClassLibrary')

import win32com.client
from pathlib import Path
import datetime
from O_Days import *
from O_Paths import *
from O_Files import *
import os
#import openpyxl
from O_MarketData import *

# Computer
path = Path()
class Excel():

    def __init__(self):

        self.Walser_Ignite_Specials = f'{path.desktop_path}/Walser Ignite Specials.xlsm'
        self.xl = win32com.client.DispatchEx('Excel.Application')
        self.refresh = self.refresh_sheet(sheet=f"{self.Walser_Ignite_Specials}")

    def refresh_sheet(self,sheet):
        print(f'Refreshing Excel Sheet:{sheet}')
        wb = self.xl.Workbooks.Open(sheet)
        ws = wb.Worksheets(1)
        wb.RefreshAll()
        print(f"Batch: {int(ws.Range('A18'))}")
        self.xl.CalculateUntilAsyncQueriesDone()
        print(f'Completed Refresh of Excel Sheet: {sheet}')
        print(f"Batch: {int(ws.Range('A18'))}")
        wb.Save()
        self.xl.DisplayAlerts = False
        self.xl.Quit()

    @staticmethod
    def split_filename_to_identify_date(filename):
        filename_strings = filename.split('_')
        date_strings = filename_strings[2].split('-')
        day_split = date_strings[2].split('.')
        date = {
            'Year': int(date_strings[0]),
            'Month': int(date_strings[1]),
            'Day': int(day_split[0])
            }
        return date

    def gather_files(self,Directory):
        list_of_valid_files = []
        for filename in os.listdir(Directory):
            today = Today()
            if 'csv#' in filename:
                filename_date = self.split_filename_to_identify_date(filename)
                file_date_valid = today.date_x_days_ago(Minus_Days=7,Year=filename_date['Year'],Month=filename_date['Month'],Day=filename_date['Day'])
                file = File()
                file.Directory = Directory
                file.FileName = filename
                if 'MarketData' in file.FileName and file_date_valid:
                    print(os.path.join(file.Directory, file.FileName))
                    valid_filenames.append(file)
                else:
                    print(filename)

        return list_of_valid_files

    def gather_market_data_files(self):
        list_of_valid_files = []
        folder_1 = self.gather_files(Directory=path.inventory_path_1)
        for f in folder_1:
            list_of_valid_files.append(f)
        folder_2 = self.gather_files(Directory=path.inventory_path_2)
        for f in folder_2:
            list_of_valid_files.append(f)
        folder_3 = self.gather_files(Directory=path.inventory_path_3)
        for f in folder_3:
            list_of_valid_files.append(f)
        folder_4 = self.gather_files(Directory=path.inventory_path_4)
        for f in folder_4:
            list_of_valid_files.append(f)

        return list_of_valid_files

    def create_list_of_vehicles(self):

        list_of_valid_files = self.gather_market_data_files()
        MarketData_Vehicles = []

        for f in list_of_valid_files:
            wb = openpyxl.load_workbook(f.File, data_only=True)
            columns = wb[f'{f.FileName}']
            row_end = len(columns['A'])
            prev_VIN = ''
            for row in columns[f'A1:U{row_end}']:
                values = []
                for cell in row:
                    values.append(cell.value)
                if values[7] != prev_VIN and values[7] != 'VIN':
                    market_data = MarketData()
                    market_data.MAKE = values[0]
                    market_data.YEAR = values[1]
                    market_data.MODEL = values[2]
                    market_data.BUILD = values[3]
                    market_data.TRIM = values[4]
                    market_data.STATUS = values[5]
                    market_data.MSRP = values[6]
                    market_data.VIN = values[7]
                    market_data.VON = values[8]
                    market_data.ALLOCATION_DATE = values[9]
                    market_data.TRANSIT_DATE = values[10]
                    market_data.GROUND_DATE = values[11]
                    market_data.SOLD_DATE = values[12]
                    market_data.LAST_STATUS_CHANGE = values[13]
                    market_data.OPTIONS = values[14]
                    market_data.EXT_COLOR = values[15]
                    market_data.INT_COLOR = values[16]
                    market_data.DAYS_IN_STOCK = values[17]
                    market_data.DEALER_TITLE = values[18]
                    market_data.DISTANCE = values[19]
                    market_data.BVS = values[20]

                    prev_VIN = market_data.VIN

                    MarketData_Vehicles.append(market_data)
        for m in MarketData_Vehicles:
            print(m)


import time as time
import numpy as np
from keras.datasets import mnist
from keras.utils.np_utils import to_categorical

def mnist_data():
    # loading the image in train and test samples
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    total_train_sample, height, width = x_train.shape
    dimension = height * width

    # preprocessing the dataset
    scale = 225.0
    x_train = x_train.reshape(total_train_sample, dimension)
    x_test = x_test.reshape(x_test.shape[0], dimension)
    x_train = x_train.astype('float32')
    x_train = x_train / scale
    x_test = x_test.astype('float32')
    x_test = x_test / scale
    y_train = to_categorical(y_train, 10)
    y_test = to_categorical(y_test, 10)

    return x_train, y_train, x_test, y_test

class Ielm_mnist():
    def __init__(self, x_train, y_train, x_test, y_test):
        self.x_train = x_train
        self.y_train = y_train
        self.x_test = x_test
        self.y_test = y_test
        self.input_neuron = self.x_train.shape[1]  # input_neuron=784
        self.output_neuron = self.y_train.shape[1]  # output_neuron=10

    # activation function:
    def sigmoid(self, x):
       return 1/(1 + np.exp(-x))

    # train module:
    def train(self,x_train,y_train) :
        start_train_process= time.process_time()
        print("-------------------------------")
        print(" Training Ielm")
        print("-------------------------------")
        E = self.y_train
        L = 0  # hidden_neuron
        while L <= 1000:
           input_weight = np.random.normal(size=[self.input_neuron,L])
           bias = np.random.normal(size=[L])
           g = np.dot(x_train, input_weight)
           g = g + bias
           H = self.sigmoid(g)
           inverse = np.linalg.pinv(H)
           beta = np.dot(inverse,y_train) #learning parameter
           L = L+50
           d = np.dot(H, beta)
           E = E - d
        total_train = x_train.shape[0]
        correct_value = 0
        for i in range(total_train):
            actual = np.argmax(y_train[i])
            predict = np.argmax(d[i])
            if predict == actual:
                 correct_value =correct_value + 1
        testing_accuracy = correct_value/total_train
        print("-------------------------------")
        print("Training accuracy:", testing_accuracy)
        print("-------------------------------")
        end_train_process = time.process_time()
        training_time =end_train_process-start_train_process
        print("Training Time:",training_time)
        print("-------------------------------")
        return L,beta

    # test module:
    def test(self, x_test, y_test, L, beta ):
        L = L-50
        start_test_process = time.process_time()
        print("-------------------------------")
        print(" Testing Ielm")
        print("-------------------------------")
        input_weight=np.random.normal(size=[self.input_neuron,L])
        bias = np.random.normal(size=[L])
        g = np.dot(x_test, input_weight)
        g = g + bias
        H = self.sigmoid(g)
        d = np.dot(H, beta)
        total_test = x_test.shape[0]
        correct_value = 0
        for i in range(total_test):
            actual = np.argmax(y_test[i])
            predict = np.argmax(d[i])
            if predict == actual:
                 correct_value = correct_value + 1
            testing_accuracy = correct_value/total_test
        print("-------------------------------")
        print("Testing accuracy:", testing_accuracy)
        print("-------------------------------")
        end_test_process = time.process_time()
        testing_time = end_test_process-start_test_process
        print("Testing Time:", testing_time)
        print("-------------------------------")

def main():
    # calling mnist dataset
    x_train, y_train, x_test, y_test = mnist_data()

    # calling the ielm class
    ielm = Ielm_mnist(x_train, y_train, x_test, y_test)
    L, beta = ielm.train(x_train, y_train)
    ielm.test(x_test, y_test, L, beta)

if __name__ == "__main__":
    main()