import base64
from http import HTTPStatus
from typing import Tuple

import cv2 as cv
import numpy as np
from flask import render_template, request, abort, jsonify, Response

from app import model, face_cascade
from app.main import main_bp

@main_bp.route('/', methods=['GET'])
def index() -> Tuple[str, int]:
    return render_template('index.html'), \
           HTTPStatus.OK

@main_bp.route('/api/image', methods=['POST'])
def classify() -> Tuple[Response, int]:
    url = request.json.get('dataURL', '')
    data = url.split(',')
    if len(data) != 2:
        abort(HTTPStatus.BAD_REQUEST)

    image = base64.b64decode(data[1])
    image = np.frombuffer(image, dtype=np.uint8)
    image = cv.imdecode(image, flags=0)

    results = []
    emotions = ('angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral')
    faces = face_cascade.detectMultiScale(image, 1.03, 5)
    for (x, y, w, h) in faces:
        roi = image[y:y + h, x:x + w]
        roi = cv.resize(roi, (48, 48))
        roi = np.expand_dims(roi, axis=0)

        predictions = model.predict(roi)
        label = emotions[np.argmax(predictions)]
        score = np.max(predictions) * 100
        results.append({
            'x': int(x),
            'y': int(y),
            'w': int(w),
            'h': int(h),
            'label': label,
            'score': int(score)
        })

    return jsonify(results), \
           HTTPStatus.OK


#!/usr/bin/env python3

###########################
# Created by Sam Shepherd #
#   Checks all nodes and  #
#   attempts to write a   #
# datafile containing all #
#        the files        #
#       13/12/2021        #
###########################

# Import modules.
import os, sys
import glob
import re
from ase.io.bundletrajectory import main
import numpy as np
import argparse

from aiida import load_profile
from aiida.common.exceptions import NotExistent
from aiida.orm import load_node
from ase.io import read, write
from ase.units import Bohr

# Load AiiDA profile
load_profile('samshepherd')

# Define the command line parser and deal with choices appropriate to the start of the script
parser = argparse.ArgumentParser(description='Provenance tracking combination script for all datafiles')
parser.add_argument('dirs', metavar='Directories', type=str, help='general directory name')
parser.add_argument('--units', metavar='Output units', help='Output units', required=True, type=str.lower, choices=['au', 'ev'], default='au')
parser.add_argument('--add', metavar='Additional label text', type=str, required=False, help='additional text to help identify labels')
parser.add_argument('--forces', metavar='Force array name', help='Name of the Force array within the Array node', type=str, required=False, default='forces')
parser.add_argument('--output', metavar='Output location', type=str, required=False, help='location to write the output file to', default=os.getcwd())
parser.add_argument('--keep_outputs', metavar='Keep intermediate `result.xyz` files', type=bool, required=False, help='Determines whether intermediate files are kept', default=False)
args = parser.parse_args()

def mainFunc(args):
    genDirs = f'{args.dirs}*'
    if args.units == 'au':
        EngVar, ForcVar = 1, 1
        posVar = Bohr
    else:
        EngVar, ForcVar = 27.211396641308, 51.422067
        posVar = 1

# Assess all directories within this directory with form of `dirs` in arguments
    allDirs = glob.glob(genDirs)
    ParentWD = os.getcwd()

    newFrames = []
    for dir in allDirs:
        nodeLabel = f'{dir}{args.add}'
        try:
            node = load_node(nodeLabel)
        except NotExistent:
            print(f'{nodeLabel} not found... consider adding to fail nodes directory')
            raise
        os.chdir(dir)
        ogFrame = read(f'{dir}.xyz')
        # Load all other required values for the transformation of the original frame
        try:
            engVal = node.outputs.output_parameters['energy']
            forNode = node.outputs.atomic_forces
        except NotExistent:
            print(f'Calculation {nodeLabel} did not return an output... consider adding it to fail nodes directory')
        forArray = forNode.get_array(args.forces)
        # If the length of the array is larger than the number of atoms, remove the top row
        if len(ogFrame.get_atomic_numbers()) < forArray.shape[0]:
            forArray = forArray[1:,:]
        if forArray.shape[1] > 3:
            forArray = forArray[:,2:]
        # Edit frame as required prior to addition of other information
        ogFrame.set_cell(ogFrame.cell / posVar)
        ogFrame.set_positions(ogFrame.positions / posVar)
        # Edit frame to include new information
        ogFrame.arrays['force'] = forArray.astype('float64') * ForcVar
        write('result.xyz', ogFrame)

        # Use a context manager to change the energy.
        resultPath = os.path.join(os.getcwd(), 'result.xyz')
        with open(resultPath, 'r') as handle:
            data = handle.readlines()

        with open('tmp', 'w+') as output:
            for line in data:
                if 'energy=' in line:
                    words = line.split()
                    for ii, word in enumerate(words):
                        if re.search('energy=', word):
                            engToChange = word
                            engIndex = words.index(word)
                            words[engIndex] = f"{engToChange.split('-')[0]}{str(engVal * EngVar)}"
                    EngString = ' '.join(words)
                    output.writelines(f'{EngString}\n')
                else:
                    output.writelines(f'{line}')

        # Do some editting of the code within this block to get the correct frame loaded and delete the old frame.
        os.system('mv tmp result.xyz')
        newFrame = read('result.xyz')
        os.system('rm result.xyz')
        write('result.xyz', newFrame)
        newFrames.append(newFrame)

        # Determine whether to delete intermediate files
        if args.keep_outputs != True:
            os.system('rm result.xyz')
        os.chdir(ParentWD)

    write(f'Kaolinite_revPBE0D3_{os.path.basename(args.output)}.{args.units}.xyz', newFrames)
    print(f'Kaolinite_revPBE0D3_{os.path.basename(args.output)}.{args.units}.xyz has been written.')

if __name__ == "__main__":
    mainFunc(args)


import sys, os
import numpy as np
import torch.nn as nn
import cv2
import matplotlib.pyplot as plt
from pathlib import Path
import torch
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data.sampler import SequentialSampler
from torch.utils.data import Dataset
from torchvision import transforms
import argparse
from tqdm import tqdm
sys.path.append('../')
from pipeline_encoders import IMU_ENCODER, TEMP_ENCODER, VIS_ENCODER
from helpers import Helpers
# from FlowNetPytorch.models import FlowNetS
from variables import RootVariables
from torch.utils.tensorboard import SummaryWriter

class FusionPipeline(nn.Module):
    def __init__(self):
        super(FusionPipeline, self).__init__()
        torch.manual_seed(2)
        self.var = RootVariables()
        self.checkpoint_path = self.var.root + checkpoint
        self.activation = nn.Sigmoid()
        self.temporalSeq = 32
        self.temporalSize = 16
        self.trim_frame_size = 150
        self.imuCheckpoint_file = 'signal_checkpoint0_' + test_folder[5:] + '.pth'
        self.frameCheckpoint_file = 'vision_checkpointAdam9CNN_' + test_folder[5:] +'.pth'

        ## IMU Models
        self.imuModel = IMU_ENCODER()
        imuCheckpoint = torch.load(self.var.root + 'datasets/' + self.var.test_folder[5:] + '/' + self.imuCheckpoint_file,  map_location="cuda:0")
        self.imuModel.load_state_dict(imuCheckpoint['model_state_dict'])
        for params in self.imuModel.parameters():
             params.requires_grad = True

        ## FRAME MODELS
        self.frameModel =  VIS_ENCODER()
        frameCheckpoint = torch.load(self.var.root + 'datasets/' + self.var.test_folder[5:] + '/' + self.frameCheckpoint_file,  map_location="cuda:0")
        self.frameModel.load_state_dict(frameCheckpoint['model_state_dict'])
        for params in self.frameModel.parameters():
            params.requires_grad = True

        ## TEMPORAL MODELS
 #       self.temporalModel = TEMP_ENCODER(self.temporalSize)

#        self.fc1 = nn.Linear(self.var.hidden_size, 2).to("cuda:2")
        self.dropout = nn.Dropout(0.35)
        self.fc0 = nn.Linear(512, 256).to("cuda:0")
        self.fc1 = nn.Linear(256, 2).to("cuda:0")
#        self.fc2 = nn.Linear(128, 2).to("cuda:2")
        ##OTHER
        self.imu_encoder_params = None
        self.frame_encoder_params = None
        self.imuBN = nn.BatchNorm1d(self.var.hidden_size*2, affine=True).to("cuda:0")
        self.frameBN = nn.BatchNorm1d(self.var.hidden_size*2, affine=True).to("cuda:0")
        self.fcBN = nn.BatchNorm1d(256).to("cuda:0")
        self.tensorboard_folder = ''

    def get_encoder_params(self, imu_BatchData, frame_BatchData):
        self.imu_encoder_params = F.relu(self.imuBN(self.imuModel(imu_BatchData.float()))).to("cuda:0")
        self.frame_encoder_params = F.relu(self.frameBN(self.frameModel(frame_BatchData.float()))).to("cuda:0")
#        self.frame_encoder_params = F.leaky_relu(self.dropout(self.downsample(self.frame_encoder_params)), 0.1).to("cuda:1")
        return self.imu_encoder_params, self.frame_encoder_params

    def fusion_network(self, imu_params, frame_params):
        return torch.cat((frame_params, imu_params), dim=1).to("cuda:0")

    def temporal_modelling(self, fused_params):
 #       newParams = fused_params.reshape(fused_params.shape[0], self.temporalSeq, self.temporalSize)
 #       tempOut = self.temporalModel(newParams.float()).to("cuda:2")
 #       gaze_pred = self.fc1(tempOut).to("cuda:2")
 #       print(fused_params, self.fc0.weight)
        gaze_pred = F.relu(self.fcBN(self.fc0(self.dropout(fused_params)))).to("cuda:0")
        gaze_pred = F.relu(self.fc1(self.dropout(gaze_pred))).to("cuda:0")
#        gaze_pred = self.fc2(self.dropout(gaze_pred)).to("cuda:2")

        return gaze_pred

    def forward(self, batch_frame_data, batch_imu_data):
        imu_params, frame_params = self.get_encoder_params(batch_imu_data, batch_frame_data)
        fused = self.fusion_network(imu_params, frame_params)
        coordinate = self.temporal_modelling(fused)

        for index, val in enumerate(coordinate):
            if coordinate[index][0] > 512.0:
                coordinate[index][0] = 512.0
            if coordinate[index][1] > 384.0:
                coordinate[index][1] = 384.0

        return coordinate

    def get_num_correct(self, pred, label):
        return torch.logical_and((torch.abs(pred[:,0]-label[:,0]) <= 100.0), (torch.abs(pred[:,1]-label[:,1]) <= 100.0)).sum().item()

    def get_original_coordinates(self, pred, labels):
        pred[:,0] *= 3.75
        pred[:,1] *= 2.8125

        labels[:,0] *= 3.75
        labels[:,1] *= 2.8125

        return pred, labels

class FINAL_DATASET(Dataset):
    def __init__(self, folder_type, imu_feat, labels):
        self.var = RootVariables()
        self.folder_type = folder_type
        self.imu_data = []
        self.indexes = []
        checkedLast = False
        for index in range(len(labels)):
            check = np.isnan(labels[index])
            imu_check = np.isnan(imu_feat[index])
            if check.any() or imu_check.any():
                continue
            else:
                self.indexes.append(index)
                self.imu_data.append(imu_feat[index])

        self.imu_data = self.standarization(self.imu_data)

        self.transforms = transforms.Compose([transforms.ToTensor()])
#        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    def standarization(self, datas):
        datas = np.array(datas)
        seq = datas.shape[1]
        datas = datas.reshape(-1, datas.shape[-1])
        rows, cols = datas.shape
        for i in range(cols):
            mean = np.mean(datas[:,i])
            std = np.std(datas[:,i])
            datas[:,i] = (datas[:,i] - mean) / std

        datas = datas.reshape(-1, seq, datas.shape[-1])
        return datas

    def __len__(self):
        return len(self.indexes) # len(self.labels)

    def __getitem__(self, index):
        f_index = self.indexes[index]
#        img = self.frames[f_index]
        img =  np.load(self.var.root + self.folder_type + '/frames_' + str(f_index) +'.npy')
        targets = self.gaze_data[f_index]
        targets[:,0] *= 512.0
        targets[:,1] *= 384.0

        return self.transforms(img).to("cuda:0"), torch.from_numpy(self.imu_data[index]).to("cuda:0"), torch.from_numpy(targets).to("cuda:0")

if __name__ == "__main__":
    #device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    var = RootVariables()
    parser = argparse.ArgumentParser()
    parser.add_argument("--sepoch", type=int, default=0)
    # parser.add_argument('--sepoch', action='store_true', help='Run model in pseudo-fp16 mode (fp16 storage fp32 math).')
    parser.add_argument("--nepoch", type=int, default=15)
    parser.add_argument("--tfolder", action='store', help='tensorboard_folder name')
    parser.add_argument("--reset_data", type=int)
    args = parser.parse_args()

    lastFolder, newFolder = None, None
    for index, subDir in enumerate(sorted(os.listdir(var.root))):
#        if 'train_BookShelf_S1' in subDir:
#            continue
        if 'train_PosterSession' in subDir:
            print(subDir)
            newFolder = subDir
            os.chdir(var.root)

            test_folder = 'test_' + newFolder[6:]
            _ = os.system('mv ' + newFolder + ' test_' + newFolder[6:])
            if lastFolder is not None:
                print('Last folder changed')
                _ = os.system('mv test_' + lastFolder[6:] + ' ' + lastFolder)

            print(newFolder, lastFolder)
            model_checkpoint = 'pipeline_checkpointAdam_' + test_folder[5:] + '.pth'
            arg = 'del'
            trim_frame_size = 150
            var.test_folder = test_folder

            pipeline = FusionPipeline()
            pipeline.tensorboard_folder = args.tfolder
            optimizer  = optim.Adam(pipeline.parameters(), lr=0.00095,  amsgrad=True)
#            lambda1 = lambda epoch: 0.85 ** epoch
#            scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda1, last_epoch=-1)
            criterion = nn.L1Loss()
            print(pipeline)
            best_test_acc = 0.0
            if Path(pipeline.var.root + 'datasets/' + test_folder[5:] + '/' + model_checkpoint).is_file():
                checkpoint = torch.load(pipeline.var.root + 'datasets/' + test_folder[5:] + '/' + model_checkpoint)
                pipeline.load_state_dict(checkpoint['model_state_dict'])
                best_test_acc = checkpoint['best_test_acc']
                # pipeline.current_loss = checkpoint['loss']
                print('Model loaded')
            #if 'BookShelf' in test_folder:
            #    utils = Helpers(test_folder, reset_dataset=1)
            #else:
            utils = Helpers(test_folder, reset_dataset=args.reset_data)
            n_epochs = 15
            imu_training, imu_testing, training_target, testing_target = utils.load_datasets()
            os.chdir(pipeline.var.root)

            for epoch in tqdm(range(args.sepoch, args.nepochs), desc="epochs"):
                if epoch > 0:
                    utils = Helpers(test_folder, reset_dataset=0)
                    imu_training, imu_testing, training_target, testing_target = utils.load_datasets()

#                ttesting_target = np.copy(testing_target)
#                timu_training = np.copy(imu_testing)
                trainDataset = FINAL_DATASET('training_images', imu_training, training_target)
                trainLoader = torch.utils.data.DataLoader(trainDataset, shuffle=True, batch_size=pipeline.var.batch_size, drop_last=True, num_workers=0)
                tqdm_trainLoader = tqdm(trainLoader)
                testDataset = FINAL_DATASET('testing_images', imu_testing,  testing_target)
                testLoader = torch.utils.data.DataLoader(testDataset, shuffle=True, batch_size=pipeline.var.batch_size, drop_last=True, num_workers=0)
                tqdm_testLoader = tqdm(testLoader)

                if epoch == 0 and 'del' in arg:
                    # _ = os.system('mv runs new_backup')
                    _ = os.system('rm -rf ' + pipeline.var.root + 'datasets/' + test_folder[5:] + '/runs/' + pipeline.tensorboard_folder)

                num_samples = 0
                total_loss, total_correct, total_accuracy = [], 0.0, 0.0
                pipeline.train()
                tb = SummaryWriter(pipeline.var.root + 'datasets/' + test_folder[5:] + '/runs/' + pipeline.tensorboard_folder)
                for batch_index, (frame_feat, imu_feat, labels) in enumerate(tqdm_trainLoader):
                    num_samples += frame_feat.size(0)
                    labels = labels[:,0,:]
                    pred = pipeline(frame_feat, imu_feat)
#                    print(pred, labels)
                    loss = criterion(pred.float(), labels.float())
                    optimizer.zero_grad()
                    loss.backward()
                    ## add gradient clipping
#                    nn.utils.clip_grad_value_(pipeline.parameters(), clip_value=1.0)
                    optimizer.step()

                    with torch.no_grad():

                        pred, labels = pipeline.get_original_coordinates(pred, labels)

#                        dist = torch.cdist(pred, labels.float(), p=2)[0].unsqueeze(dim=0)
#                        if batch_index > 0:
#                            trainPD = torch.cat((trainPD, dist), 1)
#                        else:
#                            trainPD = dist

                        total_loss.append(loss.detach().item())
                        total_correct += pipeline.get_num_correct(pred, labels.float())
                        total_accuracy = total_correct / num_samples
                        tqdm_trainLoader.set_description('training: ' + '_loss: {:.4} correct: {} accuracy: {:.3} lr:{}'.format(
                            np.mean(total_loss), total_correct, 100.0*total_accuracy, optimizer.param_groups[0]['lr']))

#                scheduler.step()
                pipeline.eval()
                with torch.no_grad():
                    tb = SummaryWriter(pipeline.var.root + 'datasets/' + test_folder[5:] + '/runs/' + pipeline.tensorboard_folder)
                    tb.add_scalar("Train Loss", np.mean(total_loss), epoch)
                    tb.add_scalar("Training Correct", total_correct, epoch)
                    tb.add_scalar("Train Accuracy", total_accuracy, epoch)

                    num_samples = 0
                    total_loss, total_correct, total_accuracy = [], 0.0, 0.0
                    dummy_correct, dummy_accuracy = 0.0, 0.0
                    for batch_index, (frame_feat, imu_feat, labels) in enumerate(tqdm_testLoader):
                        num_samples += frame_feat.size(0)
                        labels = labels[:,0,:]
                        dummy_pts = (torch.ones(8, 2) * 0.5).to("cuda:0")
                        dummy_pts[:,0] *= 1920
                        dummy_pts[:,1] *= 1080

                        pred = pipeline(frame_feat, imu_feat)
                        loss = criterion(pred.float(), labels.float())
                        pred, labels = pipeline.get_original_coordinates(pred, labels)

#                        dist = torch.cdist(pred, labels.float(), p=2)[0].unsqueeze(dim=0)
#                        if batch_index > 0:
#                            testPD = torch.cat((testPD, dist), 1)
#                        else:
#                            testPD = dist

                        total_loss.append(loss.detach().item())
                        total_correct += pipeline.get_num_correct(pred, labels.float())
                        dummy_correct += pipeline.get_num_correct(dummy_pts.float(), labels.float())
                        dummy_accuracy = dummy_correct / num_samples
                        total_accuracy = total_correct / num_samples
                        tqdm_testLoader.set_description('testing: ' + '_loss: {:.4} correct: {} accuracy: {:.3} DAcc: {:.4}'.format(
                            np.mean(total_loss), total_correct, 100.0*total_accuracy,  np.floor(100.0*dummy_accuracy)))

                tb.add_scalar("Testing Loss", np.mean(total_loss), epoch)
                tb.add_scalar("Testing Correct", total_correct, epoch)
                tb.add_scalar("Testing Accuracy", total_accuracy, epoch)
                tb.add_scalar("Dummy Accuracy", np.floor(100.0*dummy_accuracy), epoch)
                tb.close()

                if total_accuracy >= best_test_acc:
                    best_test_acc = total_accuracy
                    torch.save({
                                'epoch': epoch,
                                'model_state_dict': pipeline.state_dict(),
                                'optimizer_state_dict': optimizer.state_dict(),
                                'best_test_acc': best_test_acc,
                                }, pipeline.var.root + 'datasets/' + test_folder[5:] + '/' + model_checkpoint)
                    print('Model saved')

            lastFolder = newFolder

    # optimizer = optim.Adam([
    #                         {'params': imuModel.parameters(), 'lr': 1e-4},
    #                         {'params': frameModel.parameters(), 'lr': 1e-4},
    #                         {'params': temporalModel.parameters(), 'lr': 1e-4}
    #                         ], lr=1e-3)


# -*- coding: utf-8 -*-
"""
Created on Sun Dec 15 22:04:13 2022

@author: Daniel Mastropietro
@description: Reads FV results and plots them
"""

import runpy
runpy.run_path('../../setup.py')
#raise KeyboardInterrupt    # Use this to stop the execution here and keep working on the console

import os

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from run_FV import plot_results_fv_mc

#=================================== SINGLE SERVER ==================================
queue_system = "single-server"
resultsdir = "../../RL-002-QueueBlocking/results/FV/" + queue_system

#-- Execution parameters whose results should be read
# The values of the execution parameters are present in the names of the result files
# Store all the parameter combinations (K, J, N) and (K, J, T) that we want to show in plots, as follows:
# - For each (K, J, N) combination, a plot vs. T is shown.
# - For each (K, J, T) combination, a plot vs. N is shown.
# Use 0 for a particular parameter if there is NO result file for the K, J combination
# (e.g. K=20, J=8, N=0 indicates that there is no result for K=20, J=8 and an analysis of convergence by T (for a fixed value of N))
# (do NOT use NaN because that makes all the values in the data frame be interpreted as float (adding a `.0` at the end (e.g. 20.0)
# and this breaks reading the results file --because values are shown with no decimals in the file names)
# Entries `saveN` and `saveT` indicate whether the corresponding results for fixed N and for fixed T should be saved to a png file, respectively.
df_params = pd.DataFrame.from_dict({'K': [20, 20,
                                          40, 40,
                                              40],
                                    'J': [ 8, 16,
                                           4, 16,
                                              16],
                                    'N': [157, 87,
                                          3194, 372,
                                                5938],
                                    'T': [106, 4374,
                                          185, 1944,
                                                0],
                                    'saveN': [False, False,
                                              False, False,
                                                     False],
                                    'saveT': [False, False,
                                              False, False,
                                                     False]})
# Plots for the QUESTA paper (2023)
# We would like to show convergence results against one parameter (N or T) for different % error levels of the OTHER parameter (T or N)
df_params = pd.DataFrame.from_dict({'K': [20, 20, 20, 20,
                                          40, 40, 40],
                                    'J': [ 12, 12, 12, 12,
                                           12, 12, 12],
                                    'N': [264, 30, 11, 66,
                                          3051, 1356, 489],
                                    'T': [4157, 462, 167, 0,
                                          4157, 462, 167],
                                    'saveN': [False, False, False, False,
                                              False, False, False],
                                    'saveT': [True, False, False, False,
                                              True, False, False]})

# Minimum number of particles N and of arrival steps T to plot (to avoid clogging of the plot)
Nmin = 100
Tmin = 500
show_title = False
fontsize = 20
figsize = (10, 10)  # Figure size as if only the FV results were plotted. When we include the MC results in the plot as well, the first dimension (width) is multiplied by 2

# We store the results in a list of dictionaries of data frames
results = [dict()]*df_params.shape[0]
# We first compute the maximum values of x and of y across ALL parameter settings analyzed
xmax = 0
ymax = 0
for row in df_params.iterrows():
    case = row[0]
    params = row[1]
    K = params['K']
    J = params['J']
    N = params['N']
    T = params['T']

    print("Plotting case {} of {}...".format(row[0]+1, df_params.shape[0]))
    print("\tK={}, J={} (J/K={:.2f}), N={}, T={}".format(K, J, J/K, N, T))
    # Read the results
    if T > 0:
        # For the results as a function of N we need a fixed non-NaN positive value for T (note that `np.nan > 0` is False)
        results_file_fv_N = os.path.join(resultsdir, "estimates_vs_N_IDK{}N_K={},J={},T=[{}]_results.csv" \
                                                    .format(K, K, J, T))
    if N > 0:
        # For the results as a function of T we need a fixed non-NaN positive value for N (note that `np.nan > 0` is False)
        results_file_fv_T = os.path.join(resultsdir, "estimates_vs_T_IDK{}T_K={},J={},N=[{}]_results.csv" \
                                                    .format(K, K, J, N))
    results[case] = dict({  'analysisByN': None if np.isnan(T) else pd.read_csv(results_file_fv_N),
                            'analysisByT': None if np.isnan(N) else pd.read_csv(results_file_fv_T)})
    xmax = np.max([xmax, np.nanmax(np.nan if results[case]['analysisByN'] is None else results[case]['analysisByN']['N']),
                         np.nanmax(np.nan if results[case]['analysisByT'] is None else results[case]['analysisByT']['T'])])
    ymax = np.max([ymax, np.nanmax(np.nan if results[case]['analysisByN'] is None else results[case]['analysisByN']['Pr(FV)']),
                         np.nanmax(np.nan if results[case]['analysisByN'] is None else results[case]['analysisByN']['Pr(MC)']),
                         np.nanmax(np.nan if results[case]['analysisByT'] is None else results[case]['analysisByT']['Pr(FV)']),
                         np.nanmax(np.nan if results[case]['analysisByT'] is None else results[case]['analysisByT']['Pr(MC)'])])
    print("xmax = {}, ymax = {}".format(xmax, ymax))

    # Notes about the plots generated below:
    # - The columns defined in parameters `x` and `x2` are grouping variables that define each violin plot
    # - If violins are NOT always shown it's because the probability could not be estimated for some replications

xmax = xmax * 1.1
ymax = ymax * 1.1 * 100     # Recall the vertical axis is shown in percentage!!
xmax = None
# If needed, override the ymax value in order to make the plots comparable across different analyses (e.g. vs. N and vs. T)
ymax = None
#ymax = 0.40 * 1.1
#ymax = 0.0002 * 1.1
for row in df_params.iterrows():
    case = row[0]
    params = row[1]
    K = params['K']
    J = params['J']
    N = params['N']
    T = params['T']
    saveN = params['saveN']
    saveT = params['saveT']
    if show_title:
        titleByT = "nservers={}, K={}, J={} (J/K={:.0f}%), N={}, # Burn-in Steps={}, Min # Cycles for expectations={}" \
                    .format(nservers, K, J, J / K * 100, N, burnin_time_steps, min_num_cycles_for_expectations)
        titleByN = "nservers={}, K={}, J={} (J/K={:.0f}%), T={}, # Burn-in Steps={}, Min # Cycles for expectations={}" \
                    .format(nservers, K, J, J / K * 100, T, burnin_time_steps, min_num_cycles_for_expectations)
    else:
        titleByT = titleByN = ""

    #-- Plot results as a function of N (for which we need a positive fixed value of T)
    if T > 0:
        mask = results[case]['analysisByN']['N'] >= Nmin
        nservers = results[case]['analysisByN'].iloc[0]['nservers']
        rhos = results[case]['analysisByN'].iloc[0]['rhos']
        burnin_time_steps = results[case]['analysisByN'].iloc[0]['burnin_time_steps']
        min_num_cycles_for_expectations = results[case]['analysisByN'].iloc[0]['min_n_cycles']
        if saveT:
            figfile = os.path.join(resultsdir, f"FV-ConvN-rhos={rhos},K={K},J={J},T={T}-Violin-2Xaxis.png")
        else:
            figfile = None
        axes = plot_results_fv_mc(results[case]['analysisByN'][mask], x="N", x2="#Cycles(MC)_mean",
                                  xlabel="# particles", xlabel2="# Return Cycles to state x=J-1",
                                  xmax=xmax,
                                  ymin=0.0, ymax=ymax, plot_mc=True, splines=False,
                                  fontsize=fontsize,
                                  subset=~np.isnan(results[case]['analysisByN']['Pr(FV)']),
                                  plot_violin_only=True,
                                  title=titleByN,
                                  figsize=figsize,
                                  figmaximize=True,
                                  figfile=figfile)

    #-- Plot results as a function of T (for which we need a non-NaN positive fixed value of N)
    if N > 0:
        mask = results[case]['analysisByT']['T'] >= Tmin
        nservers = results[case]['analysisByT'].iloc[0]['nservers']
        rhos = results[case]['analysisByT'].iloc[0]['rhos']
        burnin_time_steps = results[case]['analysisByT'].iloc[0]['burnin_time_steps']
        min_num_cycles_for_expectations = results[case]['analysisByT'].iloc[0]['min_n_cycles']
        if saveN:
            figfile = os.path.join(resultsdir, f"FV-ConvT-rhos={rhos},K={K},J={J},N={N}-Violin-2Xaxis.png")
        else:
            figfile = None
        axes = plot_results_fv_mc(results[case]['analysisByT'][mask], x="T", x2="#Cycles(MC)_mean",
                                  xlabel="# arrival events", xlabel2="# Return Cycles to state x=J-1",
                                  xmax=xmax,
                                  ymin=0.0, ymax=ymax, plot_mc=True, splines=False,
                                  fontsize=fontsize,
                                  subset=~np.isnan(results[case]['analysisByT']['Pr(FV)']),
                                  plot_violin_only=True,
                                  title=titleByT,
                                  figsize=figsize,
                                  figmaximize=True,
                                  figfile=figfile)

# If we need to save the plot in a figure that is already opened, we can reference the figure number
# where the plot to be saved has been created and then use plt.savefig().
#fig = plt.figure(1)
#fig.savefig(os.path.join(resultsdir, "FV-ConvN-rho=0.7,K=20,J=12,T=4157,Nmax=4000-Violin-2Xaxis.png"))
#fig = plt.figure(7)
#fig.savefig(os.path.join(resultsdir, "FV-ConvN-rho=0.7,K=40,J=12,T=4157,Nmax=3000-Violin-2Xaxis.png"))

#----------------------
# Plot a heatmap where we see the dependency of the FV estimator of the blocking probability with the expected errors of Phi and E(T_A)
import matplotlib
from matplotlib import pyplot as plt, cm, ticker as mtick
from mpl_toolkits import mplot3d
import scipy
from Python.lib.simulators.queues import compute_nparticles_and_nsteps_for_fv_process_many_settings

df1 = pd.read_csv(os.path.join(resultsdir, "estimates_vs_N_IDK20N_K=20,J=12,T=ALLTOGETHER_results.csv"))
df2 = pd.read_csv(os.path.join(resultsdir, "estimates_vs_N_IDK40N_K=40,J=12,T=ALLTOGETHER_results.csv"))

toplot = pd.concat([df1, df2])

# Compute the NT required values for each analyzed errors in the experiments (see NT_required.xlsx)
df_NT_required = compute_nparticles_and_nsteps_for_fv_process_many_settings(   rhos=[0.7],
                                                                               K_values=[5, 10, 20, 40],
                                                                               JF_values=np.arange(0.1, 0.9, 0.1),
                                                                               error_rel=[0.05] + list(np.arange(0.1, 1.1, 0.1)))
# Add the error values to the dataset to plot
toplot = pd.merge(toplot,
         df_NT_required[['K', 'J', 'N', 'error']],
         left_on=['K', 'J', 'N'],
         right_on=['K', 'J', 'N'])
toplot.rename({'error': 'error_phi'}, axis='columns', inplace=True)

toplot = pd.merge(toplot,
         df_NT_required[['K', 'J', 'T', 'error']],
         left_on=['K', 'J', 'T'],
         right_on=['K', 'J', 'T'])
toplot.rename({'error': 'error_et'}, axis='columns', inplace=True)

# Estimation error
#####  IMPORTANT: THIS STEP HAS A HIGH IMPACT ON HOW THE HEATMAP IS SEEN!! #####
# EITHER SET MISSING VALUES OF Pr(FV) TO ZERO OR NOT... (do not set it to 0 if we want to take into account those replications where we actually estimated something reliable...)
#toplot['Pr(FV)'].fillna(0, inplace=True)
#####  IMPORTANT #####
toplot['error_fv_rel'] = toplot['Pr(FV)'] / toplot['Pr(K)'] - 1
toplot['error_fv_rel_abs'] = np.abs(toplot['error_fv_rel'])
#axes = plt.figure().subplots(1,2)
#axes[0].hist(np.log2(1 + np.abs(toplot['error_fv_rel'])), bins=30); axes[0].set_title("Histogram of Absolute relative error (log2 scale)"); axes[0].axvline(1)  # We use log2 and not log10 so that when the absolute relative error is 1 (100%) the result is also 1! :)
#axes[1].hist(-np.log10(toplot['Pr(FV)'] + 1E-20), bins=30, color="red"); axes[1].set_title("Histogram of Pr(FV) (-log10 scale)") # We use log2 and not log10 so that when the absolute relative error is 1 (100%) the result is also 1! :)

# Summarize error by each error combination
toplot_agg = toplot.groupby(['K', 'J', 'N', 'T', 'error_et', 'error_phi'])[['Pr(FV)', 'Pr(K)', 'error_fv_rel', 'error_fv_rel_abs']].agg(['count', 'mean', 'min', 'median', 'max', 'std'])
toplot_agg.reset_index(inplace=True)
#toplot_agg[['error_et', 'error_phi', 'Pr(FV)']]
#toplot_agg['Pr(FV)']['mean']
# Extract the mean of each analyzed measure (Pr(FV) and error_fv_rel)
#toplot_agg_error = pd.concat([toplot_agg[['error_et', 'error_phi']], toplot_agg.xs('mean', axis=1, level=1)], axis=1)   # Not quite
toplot_agg_mean = pd.concat([toplot_agg.xs('K', axis=1), toplot_agg.xs('J', axis=1), toplot_agg.xs('N', axis=1), toplot_agg.xs('T', axis=1),
                             toplot_agg.xs('error_et', axis=1), toplot_agg.xs('error_phi', axis=1), toplot_agg.xs('mean', axis=1, level=1),
                             toplot_agg.xs(('Pr(FV)', 'count'), axis=1)],
                            axis=1)
# Compute the absolute error of the AVERAGE of Pr(FV) (this is the same as the absolute of the error_fv_rel --which is DIFFERENT from the value stored in error_fv_rel_abs because this averages the absolute values!)
toplot_agg_mean['error_fvmean_rel_abs'] = np.abs(toplot_agg_mean['error_fv_rel'])
toplot_agg_mean['n'] = toplot_agg_mean[('Pr(FV)', 'count')]
toplot_agg_median = pd.concat([ toplot_agg.xs('K', axis=1), toplot_agg.xs('J', axis=1), toplot_agg.xs('N', axis=1), toplot_agg.xs('T', axis=1),
                                toplot_agg.xs('error_et', axis=1), toplot_agg.xs('error_phi', axis=1), toplot_agg.xs('median', axis=1, level=1),
                                toplot_agg.xs(('Pr(FV)', 'count'), axis=1)],
                                axis=1)
toplot_agg_median['error_fvmean_rel_abs'] = np.abs(toplot_agg_median['error_fv_rel'])
toplot_agg_median['n'] = toplot_agg_median[('Pr(FV)', 'count')]
plt.figure(); plt.hist(toplot_agg_mean['error_fv_rel_abs'], bins=30, alpha=0.5); plt.hist(toplot_agg_mean['error_fvmean_rel_abs'], bins=30, alpha=0.5)
## OK: the orange bars are closer to the origin than the blue bars

# Compute the overall mean and median (i.e. NOT by K)
toplot_agg_overall = toplot.groupby(['J', 'error_et', 'error_phi'])[['Pr(FV)', 'Pr(K)', 'error_fv_rel', 'error_fv_rel_abs']].agg(['count', 'mean', 'min', 'median', 'max', 'std'])
toplot_agg_overall.reset_index(inplace=True)
toplot_agg_overall[('Pr(FV)', 'cv')] = toplot_agg_overall[('Pr(FV)', 'std')] / np.sqrt(toplot_agg_overall[('Pr(FV)', 'count')]) / toplot_agg_overall[('Pr(FV)', 'mean')]

toplot_agg_overall_mean = pd.concat([toplot_agg_overall.xs('J', axis=1),
                                     toplot_agg_overall.xs('error_et', axis=1), toplot_agg_overall.xs('error_phi', axis=1), toplot_agg_overall.xs('mean', axis=1, level=1),
                                     toplot_agg_overall.xs(('Pr(FV)', 'count'), axis=1)],
                                    axis=1)
toplot_agg_overall_mean['error_fvmean_rel_abs'] = np.abs(toplot_agg_overall_mean['error_fv_rel'])
toplot_agg_overall_mean['n'] = toplot_agg_overall_mean[('Pr(FV)', 'count')]

toplot_agg_overall_median = pd.concat([ toplot_agg_overall.xs('J', axis=1),
                                        toplot_agg_overall.xs('error_et', axis=1), toplot_agg_overall.xs('error_phi', axis=1), toplot_agg_overall.xs('median', axis=1, level=1),
                                        toplot_agg_overall.xs(('Pr(FV)', 'count'), axis=1)],
                                        axis=1)
toplot_agg_overall_median['error_fvmean_rel_abs'] = np.abs(toplot_agg_overall_median['error_fv_rel'])
toplot_agg_overall_median['n'] = toplot_agg_overall_median[('Pr(FV)', 'count')]

# Coefficient of Variation of the estimated mean
toplot_agg_overall_cv = pd.concat([toplot_agg_overall.xs('J', axis=1),
                                     toplot_agg_overall.xs('error_et', axis=1), toplot_agg_overall.xs('error_phi', axis=1), toplot_agg_overall.xs(('Pr(K)', 'mean'), axis=1), toplot_agg_overall.xs('cv', axis=1, level=1),
                                     toplot_agg_overall.xs(('Pr(FV)', 'count'), axis=1)],
                                    axis=1)
toplot_agg_overall_cv['Pr(K)'] = toplot_agg_overall_cv[('Pr(K)', 'mean')]
toplot_agg_overall_cv['cv'] = toplot_agg_overall_cv['Pr(FV)']
toplot_agg_overall_cv['n'] = toplot_agg_overall_cv[('Pr(FV)', 'count')]

# Mapping of axes in the below plots:
# x <- error_et
# y <- error_phi
# Do plots for every K value (in order to analyze how the relationship with the error in Pr(FV) changes with K
# (its change should be negligible...)
def plot_error_analysis(toplot_agg_stat,
                        toplot_agg,
                        xvar="error_et", yvar="error_phi",
                        xlabel="% Error E(T_A) --> Decreasing # arrival events (T)", ylabel="% Error Phi(K) --> Decreasing # particles (N)",
                        analysis_by_K=True,
                        K_values=None,          # K values considered for each error analysis
                        p_values=None,          # Pr(K) values associated to the K values considered for each analysis
                        contour_levels=None,    # List or array containing the levels to show in the contour plot. If None, the levels are defined as `np.linspace(vmin, vamx, 5) where vmin and vmax are the minimum and maximum values of the quantity defining the heatmap.
                        plot_what="error",
                        plot_scale="log",
                        plot_times_instead_of_error=True,
                        plot_types=["colormesh"],
                        cmap="jet",
                        error_max=9,    # Maximum value that should be used for the maximum value associated to the error plots... This is used so that the color scale shown in the plot is informative; when error_max = 9, it means we plot maximum a Pr(FV) estimated 10 times larger than the true Pr(K) value.
                        fontsize=12):
    if analysis_by_K:
        probas_true = np.unique(toplot_agg_stat[['K', 'Pr(K)']], axis=0)
    else:
        probas_true = [np.mean(toplot_agg_stat['Pr(K)'])]

    for itervalues in probas_true:
        if analysis_by_K:
            K, ptrue = itervalues
            mask = toplot_agg_stat['K'] == K
        else:
            ptrue = itervalues
            mask = [True]*toplot_agg_stat.shape[0]

        # Convert the data frame to a table of the values to plot so that we can create a heatmap and a contour plot
        if plot_what == "cv":
            toplot_agg_table_cv = toplot_agg_stat[mask].pivot(xvar, yvar, 'cv')
        else:
            toplot_agg_table_error = toplot_agg_stat[mask].pivot(xvar, yvar, 'error_fv_rel')
            toplot_agg_table_error_abs = toplot_agg_stat[mask].pivot(xvar, yvar, 'error_fv_rel_abs')
            toplot_agg_table_errormean_abs = toplot_agg_stat[mask].pivot(xvar, yvar, 'error_fvmean_rel_abs')
            toplot_agg_table_proba = toplot_agg_stat[mask].pivot(xvar, yvar, 'Pr(FV)')

        # Plots
        if plot_what == "error":
            toplot_table = toplot_agg_table_error
            zvar = "error_fv_rel"
            zlabel = "Rel. error Pr(FV) (with sign)"

            # Min and max values for the color scale
            value_min = -1.0
            # Make the plots comparable by using the same maximum for the colormap (note that the minimum is common to all cases (-1 for errors and 0 for the probability))
            # Note that the maximum value is NOT filtered to the currently analyzed K! (i.e. this guarantees that the scale is the same ACROSS all K values plotted)
            value_max = min(np.ceil(np.max(toplot_agg_stat[zvar])), error_max)

            # What to plot and on what scale
            if plot_times_instead_of_error:
                toplot_table = toplot_table + 1
                value_min = value_min + 1
                value_max = value_max + 1
                zlabel = "Pr(FV) / Pr(K)"
            if plot_scale == "log":
                toplot_table = np.sign(toplot_table) * np.log2(
                    1 + np.abs(toplot_table))  # We use log2() instead of log10() so that 1 is mapped to 1
                value_min = np.sign(value_min) * np.log2(1 + value_min)
                value_max = np.sign(value_max) * np.log2(1 + value_max)
                zlabel = zlabel + " (log2)"

            # Colormap normalization that honour the vmin and vmax values
            color_norm = matplotlib.colors.Normalize(vmin=value_min,
                                                     vmax=value_max)  # The minimum relative error is 100% as the estimate cannot go below 0 (in which case we have `0/Pr(K) - 1` as relative error)
        elif plot_what == "error_abs":
            toplot_table = toplot_agg_table_error_abs
            zvar = "error_fv_rel"
            zlabel = "Abs. Rel. error Pr(FV)"

            # Min and max values for the color scale
            value_min = 0.0
            # Make the plots comparable by using the same maximum for the colormap (note that the minimum is common to all cases (-1 for errors and 0 for the probability))
            # Note that the maximum value is NOT filtered to the currently analyzed K! (i.e. this guarantees that the scale is the same ACROSS all K values plotted)
            value_max = np.ceil(np.max(toplot_agg_stat[zvar]))

            # What scale to use
            if plot_scale == "log":
                toplot_table = np.sign(toplot_table) * np.log2(
                    1 + np.abs(toplot_table))  # We use log2() instead of log10() so that 1 is mapped to 1
                value_min = np.sign(value_min) * np.log2(1 + value_min)
                value_max = np.sign(value_max) * np.log2(1 + value_max)
                zlabel = zlabel + " (log2)"

            # Colormap normalization that honour the vmin and vmax values
            color_norm = matplotlib.colors.Normalize(vmin=value_min, vmax=value_max)
        elif plot_what == "errormean_abs":
            toplot_table = toplot_agg_table_errormean_abs
            zvar = "error_fv_rel"
            zlabel = "Abs. Rel. error mean(Pr(FV))"

            # Min and max values for the color scale
            value_min = 0.0
            # Make the plots comparable by using the same maximum for the colormap (note that the minimum is common to all cases (-1 for errors and 0 for the probability))
            # Note that the maximum value is NOT filtered to the currently analyzed K! (i.e. this guarantees that the scale is the same ACROSS all K values plotted)
            value_max = np.ceil(np.max(toplot_agg_stat[zvar]))

            # What scale to use
            if plot_scale == "log":
                toplot_table = np.sign(toplot_table) * np.log2(
                    1 + np.abs(toplot_table))  # We use log2() instead of log10() so that 1 is mapped to 1
                value_min = np.sign(value_min) * np.log2(1 + value_min)
                value_max = np.sign(value_max) * np.log2(1 + value_max)
                zlabel = zlabel + " (log2)"

            # Colormap normalization that honour the vmin and vmax values
            color_norm = matplotlib.colors.Normalize(vmin=value_min, vmax=value_max)
        elif plot_what == "cv":
            toplot_table = toplot_agg_table_cv
            zvar = "cv"
            zlabel = "Coefficient of Variation of Pr(FV)"

            # Min and max values for the color scale
            value_min = 0.0
            # Make the plots comparable by using the same maximum for the colormap (note that the minimum is common to all cases (-1 for errors and 0 for the probability))
            # Note that the maximum value is NOT filtered to the currently analyzed K! (i.e. this guarantees that the scale is the same ACROSS all K values plotted)
            value_max = np.ceil(np.max(toplot_agg_stat[zvar]))

            # What scale to use
            if plot_scale == "log":
                toplot_table = np.sign(toplot_table) * np.log2(1 + np.abs(toplot_table))  # We use log2() instead of log10() so that 1 is mapped to 1
                value_min = np.sign(value_min) * np.log2(1 + value_min)
                value_max = np.sign(value_max) * np.log2(1 + value_max)
                zlabel = zlabel + " (log2)"

            # Colormap normalization that honour the vmin and vmax values
            color_norm = matplotlib.colors.Normalize(vmin=value_min, vmax=value_max)
        else:
            # The default is to plot the probability estimate
            toplot_table = toplot_agg_table_proba
            zvar = "Pr(FV)"
            zlabel = zvar

            # Min and max values for the color scale
            value_min = 0.0
            # Make the plots comparable by using the same maximum for the colormap (note that the minimum is common to all cases (-1 for errors and 0 for the probability))
            # Note that the maximum value is NOT filtered to the currently analyzed K! (i.e. this guarantees that the scale is the same ACROSS all K values plotted)
            value_max = np.ceil(np.max(toplot_agg_stat[zvar]))

            # What scale to use
            if plot_scale == "log":
                toplot_table = -np.log10(toplot_table + 1E-20)
                value_min = -np.log10(value_min + 1E-20)
                value_max = -np.log10(value_max + 1E-20)
                zlabel = zlabel + " (-log10)"

            # Colormap normalization that honour the vmin and vmax values
            color_norm = matplotlib.colors.Normalize(vmin=value_min, vmax=value_max)

        if analysis_by_K:
            title = "Plot of {}, K = {:.0f} (True Pr(K) = {:.3g})".format(zlabel, K, ptrue)
        else:
            title = "Plot of {}, Average True Pr(K) = {:.3g}, Pr(K) values = {}, K values = {}" \
                .format(zlabel, probas_true[0], "" if p_values is None else ["{:.3g}%".format(p*100) for p in sorted(p_values, reverse=True)], "" if K_values is None else sorted(K_values))

        colormap = cm.get_cmap(cmap)
        if "surface" in plot_types:
            fig = plt.figure()
            ax1 = fig.add_subplot(1, 2, 1)
            ax2 = fig.add_subplot(1, 2, 2, projection='3d')

            pc = ax1.contour(toplot_table.index, toplot_table.columns, toplot_table.T, norm=color_norm,
                             cmap=colormap)  # We transpose the table values Z because the X dimension go along the columns of Z
            # ax1.plot(toplot_agg[xvar], toplot_agg[yvar], '.', color="black", markersize=12)
            ax1.scatter(toplot_agg[xvar], toplot_agg[yvar], s=toplot_agg[('Pr(FV)', 'count')], color="black")
            ax1.set_xlabel(xlabel, fontsize=fontsize)
            ax1.set_ylabel(ylabel, fontsize=fontsize)
            ax1.set_xlim((0, 1.1))
            ax1.set_ylim((0, 1.1))
            ax1.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
            ax1.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
            plt.colorbar(pc, ax=ax1)

            # Prepare the data for the surface plot (we need to have X and Y as 2D arrays, like those generated by np.meshgrid())
            # In fact, the code below is the same as doing xx, yy = np.meshgrid(x, y, indexing='ij')
            nx = toplot_table.shape[0]
            ny = toplot_table.shape[1]
            x = np.array(toplot_table.index)
            y = np.array(toplot_table.columns)
            xx = np.repeat(x, ny).reshape(nx, ny)
            yy = np.repeat(y, nx).reshape(nx, ny, order='F')
            ax2.plot_surface(xx, yy, toplot_table)
            plt.plot(toplot_agg[xvar], toplot_agg[yvar], '.', color="black", markersize=12)
            ax2.set_xlabel(xlabel, fontsize=fontsize)
            ax2.set_ylabel(ylabel, fontsize=fontsize)
            ax2.set_xlim((0, 1.1))
            ax2.set_ylim((0, 1.1))
            ax2.xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
            ax2.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
            ax2.set_zlabel(zlabel, fontsize=fontsize)

            plt.suptitle(title)

        if "colormesh" in plot_types:
            plt.figure()
            plt.pcolormesh(toplot_table.index, toplot_table.columns, toplot_table.T, shading='gouraud', cmap=colormap,
                           norm=color_norm)
            plt.xlabel(xlabel, fontsize=fontsize)
            plt.ylabel(ylabel, fontsize=fontsize)
            plt.gca().set_xlim((0, 1.1))
            plt.gca().set_ylim((0, 1.1))
            plt.gca().xaxis.set_major_formatter(mtick.PercentFormatter(1.0))
            plt.gca().yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
            plt.title(title)
            plt.colorbar()
            # Add the points where measurements are done
            # plt.plot(toplot_agg[xvar], toplot_agg[yvar], '.', color="white", markersize=12)
            plt.scatter(toplot_agg[xvar], toplot_agg[yvar], s=toplot_agg[('Pr(FV)', 'count')]**2, color="black", alpha=0.5)
            plt.scatter(toplot_agg[xvar], toplot_agg[yvar], s=toplot_agg[('Pr(FV)', 'count')]**2, color="white", alpha=0.5)
            ct = plt.contour(toplot_table.index, toplot_table.columns, toplot_table.T,
                             levels=np.linspace(value_min, value_max, 5) if contour_levels is None else contour_levels,
                             colors="white")  # We transpose the table values Z because the X dimension go along the columns of Z
            plt.clabel(ct, fmt="%0.1f")  # clabel() specifically adds labels to levels of a contour plot
            ## On adding labels to the contour plot: https://stackoverflow.com/questions/25873681/matplotlib-contour-plot-labels-overlap-axes
            # Add the y = x line
            plt.plot([value_min, value_max], [value_min, value_max], color="black", alpha=0.5)
            plt.plot([value_min, value_max], [value_min, value_max], color="white", alpha=0.5)
            # Use a square aspect ratio
            plt.gca().set_aspect('equal')

plot_error_analysis(toplot_agg_overall_median,
                    toplot_agg_overall,
                    analysis_by_K=False,
                    K_values=[int(K) for K in np.unique(toplot['K'])],
                    p_values=np.unique(toplot['Pr(K)']),
                    contour_levels=np.array([0.5, 1.0, 1.5, 2.0, 2.5, 3.0]),
                    plot_what = "error", # "error", "error_abs", "errormean_abs", "proba" (or anything else is equivalent to "proba")
                    plot_scale = "log", # "linear", "log"
                    plot_times_instead_of_error = True,  # When plot_what = "error" (error with sign) Whether to plot how many times LARGER is Pr(FV) w.r.t Pr(K) instead of the error (as it is more intuitive)
                    plot_types = ["colormesh"], #["colormesh", "surface"] # list of one or both of "colormesh", "surface"
                    cmap = "jet",
                    error_max = 9,
                    fontsize=12)

# Coefficient of Variations for each estimate (so that we know how reliable they are)
plot_error_analysis(toplot_agg_overall_cv,
                    toplot_agg_overall,
                    analysis_by_K=False,
                    K_values=[int(K) for K in np.unique(toplot['K'])],
                    p_values=np.unique(toplot['Pr(K)']),
                    contour_levels=None,
                    plot_what = "cv", # "error", "error_abs", "errormean_abs", "proba" (or anything else is equivalent to "proba")
                    plot_scale = "log", # "linear", "log"
                    plot_times_instead_of_error = True,  # When plot_what = "error" (error with sign) Whether to plot how many times LARGER is Pr(FV) w.r.t Pr(K) instead of the error (as it is more intuitive)
                    plot_types = ["colormesh"], #["colormesh", "surface"] # list of one or both of "colormesh", "surface"
                    cmap = "jet",
                    error_max = 9,
                    fontsize=12)

# Plot split by K
plot_error_analysis(toplot_agg_median,
                    toplot_agg,
                    analysis_by_K=True,
                    K_values=[int(K) for K in np.unique(toplot['K'])],
                    p_values=np.unique(toplot['Pr(K)']),
                    contour_levels=np.array([0.5, 1.0, 1.5, 2.0, 2.5, 3.0]),
                    plot_what = "error", # "error", "error_abs", "errormean_abs", "proba" (or anything else is equivalent to "proba")
                    plot_scale = "log", # "linear", "log"
                    plot_times_instead_of_error = True,  # When plot_what = "error" (error with sign) Whether to plot how many times LARGER is Pr(FV) w.r.t Pr(K) instead of the error (as it is more intuitive)
                    plot_types = ["colormesh"], #["colormesh", "surface"] # list of one or both of "colormesh", "surface"
                    cmap = "jet",
                    error_max = 9,
                    fontsize=12)

# If we wanted to show a smoothed version of the plots
# spar = 20
# fs = scipy.interpolate.SmoothBivariateSpline(toplot_agg_mean[xvar], toplot_agg_mean[yvar], toplot_agg_mean[zvar], w=None,
#                                                  bbox=[None, None, None, None], kx=3, ky=3, s=spar, eps=1e-16)
# zz = fs.ev(xx, yy)
# toplot_agg_mean[zvar + "_smooth"] = zz.reshape(-1)
# toplot_agg_table_error_smooth = toplot_agg_mean.pivot(xvar, yvar, zvar + "_smooth")
# toplot_table_smooth = toplot_agg_table_error_smooth
#
# fig = plt.figure()
# ax1 = fig.add_subplot(1, 2, 1)
# ax2 = fig.add_subplot(1, 2, 2, projection='3d')
#
# ax1.contour(toplot_table_smooth.index, toplot_table_smooth.columns, toplot_table_smooth.T)
# ax1.set_xlabel(xvar)
# ax1.set_ylabel(yvar)
# ax2.plot_surface(xx, yy, zz)
# ax2.set_xlabel(xvar)
# ax2.set_ylabel(yvar)
# ax2.set_zlabel(zlabel)
#
# plt.figure()
# plt.pcolormesh(toplot_table_smooth.index, toplot_table_smooth.columns, toplot_table_smooth.T, shading='gouraud')
#=================================== SINGLE SERVER ==================================

#=================================== LOSS NETWORK ==================================
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from run_FV import plot_results_fv_mc

queue_system = "loss-network"
resultsdir = "../../RL-002-QueueBlocking/results/FV/" + queue_system
resultsdir = "C:/users/lundinmatlab/Desktop"

#resultsfile = "estimates_vs_N_20230316_014100_K=6,block=[4, 6],costs=[180, 600],lambdas=[1, 5],rho=[0.8, 0.6],J=[0.1, 0.6],T=[500],N_values=[80, 160, 320, 640],ProbStart=True_results.csv"

#resultsfile = "estimates_vs_N_20230316_164010_K=6,block=[4, 6],costs=[180, 600],lambdas=[1, 5],rho=[0.8, 0.6],J=[0.1, 0.6],T=[500],N_values=[80],ProbStart=False_results.csv"
## Only ONE N value

#resultsfile = "estimates_vs_N_20230316_014116_K=6,block=[4, 6],costs=[180, 600],lambdas=[1, 5],rho=[0.8, 0.6],J=[0.5, 0.5],T=[500],N_values=[80, 160, 320, 640],ProbStart=True_results.csv"
#resultsfile = "estimates_vs_N_20230316_110309_K=6,block=[4, 6],costs=[2000, 20000],lambdas=[1, 5],rho=[0.5, 0.3],J=[0.7, 0.7],T=[500],N_values=[40, 80, 160],ProbStart=True_results.csv"

resultsfile = "estimates_vs_T_20230330_233027_K=6,block=[4, 6],costs=[180, 600],lambdas=[1, 5],rho=[0.8, 0.6],J=[0.2, 0.4],N=[100],T_values=[80, 160, 320, 640],ProbStart=False_results.csv"
resultsfile = "estimates_vs_T_20230330_232952_K=6,block=[4, 6],costs=[180, 600],lambdas=[1, 5],rho=[0.8, 0.6],J=[0.2, 0.4],N=[100],T_values=[80, 160, 320, 640],ProbStart=True_results.csv"

#resultsfile = "estimates_vs_N_20230315_201909_K=6,block=[4, 6],rho=[0.8, 0.8],J=[0.5, 0.5],T=[500],N_values=[20, 40, 80],ProbStart=False_results.csv"
## Does NOT have the expected cost variable!

#--------- Read and Plot
results = pd.read_csv(os.path.join(resultsdir, resultsfile))
# Remove missing values from the estimated probabilities
for v in ['Pr(FV)', 'ExpCost(FV)', 'Pr(MC)', 'ExpCost(MC)']:
    results[v].fillna(0, inplace=True)
if "ExpCost" in results.columns:
    true_cost_var = "ExpCost"
elif "ExpCost(K)" in results.columns:
    true_cost_var = "ExpCost(K)"

# Plot results
plot_mc = True
show_title = False
if show_title:
    title = resultsfile
    fontsize = 10
else:
    title = ""
    fontsize = 20
if resultsfile[:14] == "estimates_vs_N":
    xvar = "N"; xlabel = "# particles"
elif resultsfile[:14] == "estimates_vs_T":
    xvar = "T"; xlabel = "# arrival events"
axes = plot_results_fv_mc(  results, x=xvar, x2="#Cycles(MC)_mean",
                            prob_mc="ExpCost(MC)", prob_fv="ExpCost(FV)", prob_true=true_cost_var, multiplier=1,
                            xlabel=xlabel, xlabel2="# Return Cycles to absorption set",
                            ylabel="Expected cost",
                            ymin=0.0, plot_mc=plot_mc, splines=False,
                            fontsize=fontsize,
                            figmaximize=True,
                            title=title)
#=================================== LOSS NETWORK ==================================


import graphene
from django.db.models import AutoField
from graphene_django.types import DjangoObjectType

from ingredients.models import Category, Ingredient

class CategoryType(DjangoObjectType):
    class Meta:
        model = Category

class IngredientType(DjangoObjectType):
    class Meta:
        model = Ingredient

class Query(object):
    category = graphene.Field(CategoryType,
                              id=graphene.Int(),
                              name=graphene.String())
    all_categories = graphene.List(CategoryType)

    ingredient = graphene.Field(IngredientType,
                                id=graphene.Int(),
                                name=graphene.String())
    all_ingredients = graphene.List(IngredientType)

    def resolve_all_categories(self, info, **kwargs):
        return Category.objects.all()

    def resolve_all_ingredients(self, info, **kwargs):
        return Ingredient.objects.all()

    def resolve_category(self, info, **kwargs):
        id = kwargs.get('id')
        name = kwargs.get('name')

        if id is not None:
            return Category.objects.get(pk=id)

        if name is not None:
            return Category.objects.get(name=name)

        return None

    def resolve_ingredient(self, info, **kwargs):
        id = kwargs.get('id')
        name = kwargs.get('name')

        if id is not None:
            return Ingredient.objects.get(pk=id)

        if name is not None:
            return Ingredient.objects.get(name=name)

        return None

class CreateIngredient(graphene.Mutation):
    # 입력받을 파리미터 Field 정의
    class Arguments:
        name = graphene.String(required=True)
        notes = graphene.String(required=True)
        categoryName = graphene.String(required=True)

    # 반환 Field를 정의
    success = graphene.Boolean()

    # 실행할 Mutation 정의
    def mutate(root, info, **kwargs):
        tempCategoryName = kwargs.get('categoryName')

        try:
            Category.objects.get(name=tempCategoryName)
        except:
            Category( name=tempCategoryName).save()

        categoryObj = Category.objects.get(name=tempCategoryName)

        ingredient = Ingredient(
            name=kwargs.get('name'),
            notes=kwargs.get('notes'),
            category=categoryObj
        )

        # db에 저장
        ingredient.save()

        # 결과 반환
        return CreateIngredient(
            success=True
        )

# update할 필드 정의

class UpdateIngredient(graphene.Mutation):

    class Arguments:
        update_id = graphene.ID(required=True)
        name = graphene.String(required=True)
        notes = graphene.String(required=True)
        category = graphene.Int(required=True)

    # 반환값
    success = graphene.Boolean()

    def mutate(root, info, update_id, name, notes, category):

        # 조회
        model = Ingredient.objects.get(id=update_id)

        # 데이터 수정
        if name is not None:
            model.name = name

        if notes is not None:
            model.notes = notes

        if category is not None:
            model.category = Category( id=category)

        model.save()
        success = True

        return UpdateIngredient(
            success=success
        )

# mutation Field 정의
class Mutation(graphene.ObjectType):
    create_ingredient = CreateIngredient.Field()
    update_ingredient = UpdateIngredient.Field()


""" Make a plot of 1 month of solar, wind and T2m data in ERA5 and CPDN data """

import numpy as np
from netCDF4 import Dataset
import matplotlib.pyplot as plt
from read_gridded_data_functions import load_country_weather_data,read_spatial_dimensions
from matplotlib import gridspec

month = 3
country_names = ['United Kingdom','Ireland']
variable_names = ['t2m','u10','ssrd']
#title = [r'2m Temperature January 2018 $(^{\circ}C)$',r'10m wind speed January 2018 $(ms^{-1})$',r'Incoming solar radiation January 2018 $(Wm^{-2})$']
title = [r'10m wind speed $(^{\circ}C)$']
ylim = [ [-2,11], [2,12],[0,200] ]
var_name = 'u10'

# set up figure for plotting
plt.figure(figsize=(16,11))
gs = gridspec.GridSpec(2,2)

def convert_to_windpower(wind_speed_data,power_curve_file):

    """
    This function takes the ERA5 reanalysis data, loads it and applied a
    country mask (ready for conversion to energy) it then returns
    the array (of original size) with all irrelvelant gridpoints
    set to zeros.

    You will need the shpreader.natural_earth data downloaded
    to find the shapefiles.

    Args:

        gridded_wind_power (array): wind power capacity factor data, dimensions
            [time,lat,lon]. Capacity factors range between 0 and 1.

        power_curve_file (str): The filename of a .csv file
            containing the wind speeds (column 0) and capacity factors
            (column 2) of the chosen wind turbine.

    Returns:

        wind_power_cf (array): Gridded wind Power capacity factor
            data, dimensions [time,lat,lon]. Values vary between 0 and 1.

    """

    # first load in the power curve data
    pc_w = []
    pc_p = []

    with open(power_curve_file) as f:
        for line in f:
            columns = line.split()
            #print columns[0]
            pc_p.append(np.float(columns[1]))
            pc_w.append(np.float(columns[0]))  # get power curve output (CF)

    # convert to an array
    power_curve_w = np.array(pc_w)
    power_curve_p = np.array(pc_p)

    #interpolate to fine resolution.
    pc_winds = np.linspace(0,50,501) # make it finer resolution
    pc_power = np.interp(pc_winds,power_curve_w,power_curve_p)

    reshaped_speed = wind_speed_data.flatten()
    test = np.digitize(reshaped_speed,pc_winds,right=False) # indexing starts
    #from 1 so needs -1: 0 in the next bit to start from the lowest bin.
    test[test ==len(pc_winds)] = 500 # make sure the bins don't go off the
    #end (power is zero by then anyway)
    wind_power_flattened = 0.5*(pc_power[test-1]+pc_power[test])

    wind_power_cf = np.reshape(wind_power_flattened,(np.shape(wind_speed_data)))

    return(wind_power_cf)

def make_plot(t,variable,ylim,title='',xlabel='',ylabel='',fontsize=18,color='r'):
    print(variable.shape)
    plt.title(title,fontsize=fontsize)
    plt.plot(t,variable,color=color,linewidth=2)
    plt.xlabel(xlabel,fontsize=fontsize)
    plt.ylabel('Wind capacity factor',fontsize=fontsize)
    plt.xticks(fontsize=fontsize)
    plt.yticks(fontsize=fontsize)
    #plt.xlim(xlim[0],xlim[1])
    #print(ylim)
    plt.ylim(0,1)
    plt.xlim(0,15)

for i, country_name in enumerate(country_names): # np.arange(8):

    power_curve_file = '/gws/pw/j05/cop26_hackathons/oxford/Group_folders/group_7/energy_conversion/group_7/inputs/power_onshore.csv'

    # read in ERA5 data
    ERA5_dir = '/gws/pw/j05/cop26_hackathons/oxford/Data/ERA5_data_EU_domain/field_set_1/'
    file_name = 'ERA5_1hr_field_set_1_2018_'+str(month).zfill(2)+'.nc'
    var_ERA5, country_mask_ERA5 = load_country_weather_data(country_name,ERA5_dir,file_name,var_name)

    # if looking at wind speed, read in v wind too
    if var_name == 'u10':
        V10_ERA5, _ = load_country_weather_data(country_name,ERA5_dir,file_name,'v10')
        var_ERA5 = np.sqrt(var_ERA5**2 + V10_ERA5**2)

    # calculate hourly and monthly country means
    ERA5_var_mean = np.nanmean(np.nanmean(var_ERA5,axis=2),axis=1)
    ERA5_var_month = np.mean(ERA5_var_mean)
    ERA5_wp = convert_to_windpower(ERA5_var_mean,power_curve_file)

    # Read in CPDN
    CPDN_dir = '/gws/pw/j05/cop26_hackathons/oxford/Group_folders/group_7/cpdn_example_data/remap_ERA5/'
    if var_name == 't2m':
        file_name = 'tas_1hrly_mean_a000_2018-01_2018-12.nc'
        var_name_CPDN = 'item3236_1hrly_mean'
    elif var_name == 'u10':
        file_name = '10mwd_1hrly_mean_a000_2018-01_2018-12.nc'
        var_name_CPDN = 'item3249_1hrly_mean'
    elif var_name == 'ssrd':
        file_name = 'rsds_1hrly_mean_a000_2018-01_2018-12.nc'
        var_name_CPDN = 'field203'

    if var_name == 'ssrd': time_name='time0'
    else: time_name = 'time1'
    var_CPDN, country_mask_CPDN = load_country_weather_data(country_name,CPDN_dir,file_name,var_name_CPDN,chosen_months=(month),time_name=time_name)
    if var_name == 't2m': var_CPDN = var_CPDN - 273.15 # convert to deg Celsius

    # calculate hourly and monthly country means
    CPDN_var_mean = np.nanmean(np.nanmean(var_CPDN[:,0,:,:],axis=2),axis=1)
    CPDN_var_month = np.mean(CPDN_var_mean)
    CPDN_wp = convert_to_windpower(CPDN_var_mean,power_curve_file)

    title = [country_name+' ERA5 wind capacity factor',country_name+' CPDN wind capacity factor']

    ax = plt.subplot(gs[i,0])
    #if i==0: plt.text(15,12,'ERA5',fontsize=25) #,transform=ax.transAxes)
    t = np.linspace(0,31,744)
    if country_name == 'Ireland': xlabel = 'Day'
    else: xlabel = ''
    make_plot(t,ERA5_wp,ylim[i],title=title[0],color='royalblue',xlabel=xlabel)

    plt.subplot(gs[i,1])
    #if i==0: plt.text(15,12,'CPDN',fontsize=25)
    if country_name == 'Ireland':
        xlabel = 'Day'
    else: xlabel = ''
    t = np.linspace(0,30,720)
    make_plot(t,CPDN_wp,ylim[i],title=title[1],color='red',xlabel=xlabel)

figure_name = 'daily_variability_Ire_GB_10m_wind_speed.png'
print('saving to %s' % (figure_name))
plt.savefig(figure_name,bbox_inches='tight')


import math
import json
import clustering
import numpy as np
import solution
import fetch
import os
import test_p as testf
import cluster

class Project:
    def __init__(self, **kwargs):
        self.tests = {}
        self.all_solutions = {}
        self.name = 'NO NAME'
        self.cached_solutions = None
        self.skip_count = 0
        self.clusters = {}
        self.cluster_size = 0
        self.dimentions = None
        self.output = None
        self.is_tl_hidden = True
        self.is_rt_hidden = True
        self.is_wa_hidden = True
        self.is_changed = False
        self.sources_path = ''
        self.tests_path = ''

        if 'file' in kwargs:
            data = json.load(kwargs['file'])
            self.clusters = dict(
                (key, [cluster.Cluster(obj=c) for c in data['clusters'][key]]) for key in data['clusters']
            )
            self.all_solutions = dict((e['name']['file'], solution.Solution(self, obj=e)) for e in data['solutions'])
            self.tests = dict((e['name'], testf.Test(obj=e)) for e in data['tests'])
            self.is_tl_hidden = data.get('is_tl_hidden', True)
            self.is_rt_hidden = data.get('is_rt_hidden', True)
            self.is_wa_hidden = data.get('is_wa_hidden', True)
            self.cluster_size = data.get('cluster_size', 0)
            self.name = data.get('name', 'NO NAME')
            self.sources_path = data.get('sources_path',
                                         os.path.split(list(self.all_solutions.values())[0].filepath)[0])
            self.tests_path = data.get('tests_path',
                                       os.path.split(list(self.tests.values())[0].name)[0])

        if 'output' in kwargs:
            self.output = kwargs['output']

    def change_hidden_flags(self, tl=None, rt=None, wa=None):
        print('change_hidden_flags', tl, rt, wa)
        if tl is not None:
            self.is_tl_hidden = tl
        if rt is not None:
            self.is_rt_hidden = rt
        if wa is not None:
            self.is_wa_hidden = wa

        if tl is not None or rt is not None or wa is not None:
            self.is_changed = True
            self.drop_cache()

    def get_solutions(self):
        if self.cached_solutions is not None:
            return self.cached_solutions

        solutions = self.all_solutions
        if self.is_tl_hidden:
            solutions = dict((k, s) for k, s in solutions.items() if len(s.tl) == 0)

        if self.is_rt_hidden:
            solutions = dict((k, s) for k, s in solutions.items() if len(s.rt) == 0)

        if self.is_wa_hidden:
            solutions = dict((k, s) for k, s in solutions.items() if s.meta['status'] == 'PASSED_TESTS')

        self.cached_solutions = solutions
        return self.cached_solutions

    def drop_cache(self):
        self.cached_solutions = None
        self.is_changed = True

    def current_cluster_name(self, count=None):
        if count is None:
            count = self.cluster_size
        return "{}_{}_{}_{}".format(self.is_rt_hidden, self.is_wa_hidden, self.is_tl_hidden, count)

    def update_current_cluster_name(self, count):
        self.cluster_size = count

    def get_clusters(self):
        if self.cluster_size == 0:
            return []
        return self.clusters[self.current_cluster_name()]

    def is_tl_occured(self):
        for solution in self.all_solutions:
            if len(solution.tl) > 0:
                return True
        return False

    def is_rt_occured(self):
        for solution in self.all_solutions:
            if len(solution.rt) > 0:
                return True
        return False

    def size(self):
        return len(list(self.get_solutions().values()))

    def cluster_count(self):
        return max(len(self.get_clusters()), 1)

    def calculate_dimentions(self):
        def default(val):
            if math.isnan(val):
                return 1.0
            return val

        data = sorted(
            [
                {'i': i,
                 'av': default(np.median(
                     list(filter(lambda e: e != 1.0, [s.times[i] for s in self.get_solutions().values()]))))
                 } for i in range(self.get_count_tests())
            ],
            key=lambda x: x['av'], reverse=True
        )

        return data

    def sort_labels(self, labels):
        uniq, count = np.unique(labels, return_counts=True)
        data = list(map(lambda x: x[0], sorted(zip(uniq, count), key=lambda e: e[1], reverse=True)))
        labels = [data.index(e) for e in labels]

        return labels

    def get_dimentions(self):
        if self.dimentions is None:
            self.dimentions = self.calculate_dimentions()

        #print('get_dimentions', self.dimentions)
        return self.dimentions

    def get_test_id(self, testname):
        assert testname in self.tests, 'Cannot find test for file {}'.format(testname)

        return self.tests[testname].id

    def clusterize(self, count):
        if self.current_cluster_name(count) in self.clusters:
            self.update_current_cluster_name(count)
            self.is_changed = True
            return

        self.update_current_cluster_name(count)
        labels, centers = clustering.clusterize(self, kmax=count)
        labels = self.sort_labels(labels)
        self.is_changed = True
        cluster_name = self.current_cluster_name()
        self.clusters[cluster_name] = [
            cluster.Cluster(
                idx=idx,
                name='cluster {}'.format(idx),
                description='',
                center=center.tolist()
            ) for idx, center in enumerate(centers)
        ]
        for idx, label in enumerate(labels):
            self.clusters[cluster_name][label].elements.append(idx)

    def get_labels(self):
        labels = [0 for i in range(len(self.get_times()))]
        for cluster in self.get_clusters():
            for idx in cluster.elements:
                labels[idx] = cluster.id

        return labels

    def get_cluster(self, idx):
        return list(map(lambda x: x[0], filter(lambda x: x[1] == idx, zip(self.get_solutions().values(),
                                                                          self.get_labels()))))

    def get_cluster_info(self, idx):
        return self.get_clusters()[idx]

    def get_count_tests(self):
        return len(list(self.tests))

    def get_times_by_test(self, idx):
        #print('cluster {} got {}'.format(idx, self.get_cluster(idx)))
        time = list(map(lambda x: x.times[self.skip_count:], self.get_cluster(idx)))

        return [[el[key - self.skip_count] for el in time] for key in range(self.skip_count, self.get_count_tests())]

    def get_times(self):
        return list(map(lambda s: s.times[self.skip_count:], self.get_solutions().values()))

    def update_tests(self, tests):
        new_tests = []
        for testname in tests:
            file_md5 = testf.md5(testname)

            if testname not in self.tests:
                test = testf.Test(name=testname, id=self.get_count_tests(), file_md5=file_md5)
                self.tests[testname] = test
                self.is_changed = True
                new_tests.append(test)
            else:
                test = self.tests[testname]
                if test.md5 != file_md5:
                    new_tests.append(test)
                    self.is_changed = True
                    test.md5 = file_md5
                    for solution_name in self.all_solutions.keys():
                        solution = self.all_solutions[solution_name]
                        solution.times[test.id] = None
                        if test.id in solution.rt:
                            solution.rt.remove(test.id)
                        if test.id in solution.tl:
                            solution.tl.remove(test.id)

        return new_tests

    def update_sources(self, sources):
        new_souces = []
        for source in sources:
            if source['file'] not in self.all_solutions:
                new_souces.append(source)
        return new_souces

    def get_solution(self, source):
        if source['file'] not in self.all_solutions:
            s = solution.Solution(self, filename=source['file'])
            self.all_solutions[source['file']] = s
            self.drop_cache()

        return self.all_solutions[source['file']]

    def get_sources(self):
        result = []
        for e in self.all_solutions.values():
            result.append(e.get_source())
        return result

    def save(self):
        assert self.output is not None
        fetch.is_running.clear()
        file = open(self.output, 'w')
        file.seek(0)
        file.truncate()
        obj = {key: [c.dump() for c in self.clusters[key]] for key in self.clusters.keys()}
        json.dump({
            'name': self.name,
            'is_tl_hidden': self.is_tl_hidden,
            'is_rt_hidden': self.is_rt_hidden,
            'is_wa_hidden': self.is_wa_hidden,
            'cluster_size': self.cluster_size,
            'tests': [t.dump() for t in self.tests.values()],
            'solutions': [s.dump() for s in self.all_solutions.values()],
            'clusters': {key: [c.dump() for c in self.clusters[key]] for key in self.clusters.keys()},
        }, file, sort_keys=True, indent=4)
        file.flush()
        file.close()
        self.is_changed = False
        fetch.is_running.set()


import numpy as np
import pandas as pd
import nltk
import sklearn
import requests
import re
import os
import json
from sklearn import svm
from sklearn import tree
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import LinearSVC
from nltk.classify.scikitlearn import SklearnClassifier
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.parse import stanford
from nltk.parse.corenlp import CoreNLPServer
from nltk.parse.corenlp import CoreNLPParser
from nltk.parse.corenlp import CoreNLPDependencyParser
import stanza
import StanfordDependencies
from pprint import pprint
#from pycorenlp.corenlp import StanfordCoreNLP
from stanfordcorenlp import StanfordCoreNLP

#nltk.download('stopwords')
#nltk.download('punkt')
#nltk.download('averaged_perceptron_tagger')
#https://towardsdatascience.com/machine-learning-nlp-text-classification-using-scikit-learn-python-and-nltk-c52b92a7c73a
#https://www.nltk.org/api/nltk.classify.scikitlearn.html

def term_sentence(sentence):
    terms=[]
    tokenize = sent_tokenize(sentence)
    for i in tokenize:
        # Word tokenizers is used to find the words
        # and punctuation in a string
        wordsList = nltk.word_tokenize(i)

        tagged = nltk.pos_tag(wordsList)
        for t in tagged:
            terms.append(t[0])
    return terms

def tag_sentence(sentence):
    tags = []
    terms=[]
    regex=""
    POS_Parser = getTags()
    for t in range(len(POS_Parser)):
        regex+=POS_Parser[t]
        regex+="|"
    nlp = StanfordCoreNLP('http://localhost', port=9002)
    stanford = nlp.parse(sentence)
    tags = re.findall(regex, stanford)
    tags.remove('')
    nlp.close()
    return tags

def getDictionary():
    file1 =open("./input/terms.txt","r")
    unigrams=[]
    for l in file1.readlines():
        unigrams.append(l.strip())
    return unigrams

def getBigrams(data):
    bigrams=[]
    original_size=len(data['Headline'])
    for f in range(original_size):
        s = data['Headline'][f]
        nltk_tokens = nltk.word_tokenize(s)
        terms = list(nltk.bigrams(nltk_tokens))
        for t in terms:
            bigrams.append(t)
    return bigrams

def getTags():
    tags=[]
    file1=open("./input/POS_Tags.txt","r")
    for l in file1.readlines():
        tags.append(l.strip())
    return tags

def getLabelsFeatures(dataframe):
    label_names = ['Complete','Incomplete']
    label = dataframe['C/I']
    feature_names = ['Headline','C/I']
    features = dataframe[feature_names]
    return label_names, label, feature_names, features

def getALLData(filename, filename2):
    data=pd.read_csv(filename).dropna()
    data2=pd.read_csv(filename2).dropna()
    all_data = pd.concat([data, data2], axis=0).drop('Answer Portions',axis=1)
    return all_data

def getData(filename):
    data=pd.read_csv(filename)
    data=data.dropna()
    training_data=data[:60]
    testing_data=data[60:]

    return training_data, testing_data
def getDates(filename):
    data=pd.read_csv(filename)
    dates=[]
    data = data.dropna(subset=['Full Text Link'])
    for l in data['Full Text Link']:
        date = find_date(l)
        dates.append(date)
    data['Date'] = dates
    data.to_csv(filename)

#https://www.geeksforgeeks.org/part-speech-tagging-stop-words-using-nltk-python/
#https://www.geeksforgeeks.org/python-ways-to-remove-duplicates-from-list/

def getTagsDataFrame(data,pos_tags):
    data2=data.copy().drop('Headline',axis=1).drop('C/I',axis=1)
    stop_words = set(stopwords.words('english'))
    columns=[]
    all_terms=[]
    original_size=len(data['Headline'])
    for p in pos_tags:
        probs=[]
        columns.append(p)
        for f in range(original_size):
            s = data['Headline'][f]
            tags=tag_sentence(s)
            terms=term_sentence(s)
            terms_num=len(terms)
            tf = tags.count(p)
            ntf = tf/terms_num
            probs.append(ntf)
        data2[p]=probs
    tags_df=data2[columns]
    return tags_df

def getTermsDataFrame(data,unigrams):
    data2=data.copy().drop('Headline',axis=1).drop('C/I',axis=1)
    stop_words = set(stopwords.words('english'))
    columns=[]
    all_terms=[]
    original_size=len(data['Headline'])
    for u in unigrams:
        probs=[]
        columns.append(u)
        for f in range(original_size):
            s = data['Headline'][f]
            terms=term_sentence(s)
            terms_num=len(terms)
            tf = terms.count(u)
            ntf = tf/terms_num
            probs.append(ntf)
        data2[u]=probs
    unigrams_df=data2[columns]
    return unigrams_df

def getBigramsDataFrame(data,bigrams):
    data2=data.copy().drop('Headline',axis=1).drop('C/I',axis=1)
    stop_words = set(stopwords.words('english'))
    columns=[]
    all_terms=[]
    original_size=len(data['Headline'])
    for b in bigrams:
        probs=[]
        columns.append(b)
        for f in range(original_size):
            s = data['Headline'][f]
            terms=term_sentence(s)
            nltk_tokens = nltk.word_tokenize(s)
            terms = list(nltk.bigrams(nltk_tokens))
            terms_num=len(terms)
            tf = terms.count(b)
            ntf = tf/terms_num
            probs.append(ntf)
        data2[b]=probs
    bigrams_df=data2[columns]
    return bigrams_df

if __name__=="__main__":
    complete_training,complete_testing = getData('NLP_Project_Raw_Data_Complete.csv')
    incomplete_training,incomplete_testing = getData('NLP_Project_Raw_Data_Incomplete.csv')

    complete = pd.read_csv('NLP_Project_Raw_Data_Complete.csv').dropna()
    Incomplete = pd.read_csv('NLP_Project_Raw_Data_Incomplete.csv').dropna()

    all_data = pd.concat([complete, Incomplete], axis=0).drop('Answer Portions',axis=1).reset_index()
    #print(all_data)
    label_names = ['Complete','Incomplete']
    labels = all_data['C/I']
    headlines = all_data['Headline'] # POS Tag
    #https://www.geeksforgeeks.org/part-speech-tagging-stop-words-using-nltk-python/
    #https://stackabuse.com/python-for-nlp-parts-of-speech-tagging-and-named-entity-recognition/
    #https://www.nltk.org/book/ch05.html

    tags_df=getTagsDataFrame(all_data,getTags())
    results=pd.concat([headlines, tags_df], axis=1)

    terms_df = getTermsDataFrame(all_data,getDictionary())
    results=pd.concat([results, terms_df], axis=1)

    bigrams_df=getBigramsDataFrame(all_data,getBigrams(all_data))
    results=pd.concat([results, bigrams_df], axis=1)

    results=pd.concat([results, labels], axis=1)

    results.to_csv("results.csv")


import os
import discord
from discord.ext import commands
from dotenv import load_dotenv

load_dotenv(dotenv_path="config")
bot = commands.Bot(command_prefix="!", description="Bot pour Réviser")

@bot.event
async def on_ready():
    print("Ready!")

#lien utiles pour réviser
@bot.command()
async def link(ctx):
    embed = discord.Embed(title = "Les Liens utiles", color = 0x334FFF, description='[maths-et-tiques](https://www.maths-et-tiques.fr/)\n[autre site](https://nirbose.fr/)')
    embed.set_footer(text = "fin de la leçon !")
    await ctx.send(embed = embed)

#aller sur l'ent de stephane hesssel
@bot.command()
async def ent(ctx):
    embed = discord.Embed(title = "mon Etablissement", color = 0x334FFF, description='[ENT Stephane_Hessel](https://stephane-hessel.mon-ent-occitanie.fr/)')
    embed.set_image(url="https://stephane-hessel.mon-ent-occitanie.fr/lectureFichiergw.do?ID_FICHIER=1506290041830")
    await ctx.send(embed = embed)

#automatismes
@bot.command()
async def base(ctx):
    embed = discord.Embed(title = "Les automatismes",url="https://www.maths-et-tiques.fr/index.php/cours-maths/niveau-terminalet#A1",color = 0x334FFF)
    embed.set_image(url="https://i.ytimg.com/vi/QJwGIDnFKnA/maxresdefault.jpg")
    embed.set_footer(text = "fin de la leçon !")
    await ctx.send(embed = embed)

#suite
@bot.command()
async def suite(ctx):
    embed = discord.Embed(title = "Les suites",url="https://www.maths-et-tiques.fr/telech/19Suites1TM.pdf",color = 0x334FFF)
    embed.set_image(url="https://cdn.discordapp.com/attachments/752135228528787486/922930626603143198/Capture_decran_2021-12-21_201534.png")
    embed.set_footer(text = "fin de la leçon !")
    await ctx.send(embed = embed)

#
# @bot.command()
# async def suite(ctx):
#     embed = discord.Embed(title = "Les suites",url="https://www.maths-et-tiques.fr/telech/19Suites1TM.pdf",color = 0x334FFF)
#     embed.set_image(url="https://cdn.discordapp.com/attachments/752135228528787486/922930626603143198/Capture_decran_2021-12-21_201534.png")
#     embed.set_footer(text = "fin de la leçon !")
#     await ctx.send(embed = embed)

bot.run(os.getenv("TOKEN"))


import os
import subprocess
import tempfile

import pathlib
dir_ = pathlib.Path(__file__).parent.resolve()

def check_contact_from_file(pairs_list_file, thresh_max_dist=8, thresh_min_pairs=10, exe_path=f'{dir_}/check_contact.exe'):
    if not os.path.exists(exe_path):
        raise FileNotFoundError('check contact executable not found!')

    with tempfile.NamedTemporaryFile('w+b') as tf:
        cmd = f'{exe_path} {pairs_list_file} {tf.name} {thresh_max_dist} {thresh_min_pairs}'
        exe_result = subprocess.run(cmd.split())
        if exe_result.returncode != 0:
            raise RuntimeError(f'encountered error from {exe_path}: {exe_result.stderr} {exe_result.stdout}')

        tf.seek(0)

        results = []
        for line in tf:
            results.append(line.strip() == b'1')

        return results

def check_contact_many(pairs_list, thresh_max_dist, thresh_min_pairs, exe_path=f'{dir_}/check_contact.exe'):
    with tempfile.NamedTemporaryFile('w') as tf:
        for chain1_path, chain2_path in pairs_list:
            tf.write(f'{chain1_path} {chain2_path}\n')
        tf.seek(0)
        return check_contact_from_file(tf.name, thresh_max_dist, thresh_min_pairs, exe_path)

def check_contact_pair(chain1_path, chain2_path, thresh_max_dist=8, thresh_min_pairs=10, exe_path=f'{dir_}/check_contact.exe'):
    with tempfile.NamedTemporaryFile('w') as tf:
        tf.write(f'{chain1_path} {chain2_path}\n')
        tf.seek(0)
        result = check_contact_from_file(tf.name, thresh_max_dist, thresh_min_pairs, exe_path)
        return result[0]