import numpy as np
from scipy.stats import invwishart, multivariate_normal, invgamma
from scipy.linalg import cholesky

class canary:

    def __init__(self, data, ninv, diag_noise=True,
                 scale_prior_aff=None, df_prior_aff=None, scale_prior_unaff=None,
                 df_prior_unaff=None):

        """
        Class that holds the data and does the Gibbs sampling.

        Parameters:
            data (array_like):
                Array of data from which the inference is drawn.
                Should have shape (num_draw, dat_dim), where num_draw is the
                number of multivariate draws in question and dat_dim is the
                dimension of each multivariate draw.

            ninv (array_like):
                Inverse of the noise covariance matrix. If diag_noise is True
                then this may have shape (num_draw, dat_dim), otherwise this should
                have shape (num_draw, dat_dim, dat_dim). If diag_noise is True and a
                shape of (num_draw, dat_dim, dat_dim) is supplied, will extract diagonal entries.

            diag_noise (bool):
                Whether the noise covariance matrix is diagonal or not. Will
                make the calculations much faster if True.

            scale_prior_aff (array_like):
                Scale matrix for the inverse Wishart prior on the systematic
                covariance. Default is identity.

            df_prior_aff (int):
                Degrees of freedom parameter for the inverse Wishart prior on
                the systematic covariance. Default is dat_dim + 1, which is the
                lowest allowed value while still being a proper prior. Higher
                value creates a sharper prior.

            df_prior_unaff (int):
                Degrees of freedom parameter for the inverse Gamma prior on the
                unaffected data. A higher number creates a sharper prior. This
                prior should be fairly sharp around a small value so that it
                represents a very weak systematic (i.e. no systematic).

            scale_prior_unaff (float):
                Scale parameter for the inverse Gamma prior on the unaffected
                data. This should be set in correspondence with df_prior unaff
                such that the mode of the prior is on a small value relative to
                the noise variance.
        """
        self.data = np.copy(data)
        if len(data.shape) != 2:
            raise ValueError("Supplied data array must have dimension of 2.")
        self.num_draw, self.dat_dim = data.shape

        self.diag_noise = diag_noise
        self.ninv, self.ninv_cho = self._setup_noise(ninv, diag_noise)

        if scale_prior_aff is None: # uninformative
            self.scale_prior_aff = np.eye(self.dat_dim)
        else:
            self.scale_prior_aff = scale_prior_aff
        if df_prior_aff is None: # uninformative
            self.df_prior_aff = self.dat_dim + 1
        else:
            self.df_prior_aff = df_prior_aff

        if df_prior_unaff is None:
            self.df_prior_unaff = 1
        else:
            self.df_prior_unaff = df_prior_unaff
        if scale_prior_unaff is None:
            self.scale_prior_unaff = 1e-4 * self.df_prior_unaff
        else:
            self.scale_prior_unaff = scale_prior_unaff

    def _setup_noise(self, ninv, diag_noise):
        """
        Set up the noise attributes during initialization.

        Parameters:
            ninv (array_like):
                Inverse of the noise covariance matrix. If diag_noise is True
                then this may have shape (dat_dim), otherwise this should
                have shape (dat_dim, dat_dim). If diag_noise is True and a
                shape of (dat_dim, dat_dim) is supplied, will extract diagonal entries.

            diag_noise (bool):
                Whether the noise covariance matrix is diagonal or not. Will
                make the calculations much faster if True.

        Returns:
            ninv (array_like):
                Reshaped noise inverse according to diag_noise argument.
            ninv_cho (array_like:
                Cholesky decomposition of the inverse noise covariance.
                Calculated with elementwise square root if diag_noise,
                else uses scipy.linalg.cholesky.
            ninvd (array_like:
                Inverse-noise-covariance weighted data.
        """
        valid_shapes = ((self.num_draw, self.dat_dim,), (self.num_draw, self.dat_dim, self.dat_dim))
        if ninv.shape not in valid_shapes:
            raise ValueError(f"ninv shape, {ninv.shape}, does not match data shape, "
                             f"{self.data.shape}. Check inputs.")

        if diag_noise:
            if len(ninv.shape) == 3:
                ninv_ret = np.zeros(self.num_draw, self.dat_dim)
                for drind in range(self.num_draw):
                    ninv_ret[drind] = np.diag(ninv[drind])
            else:
                ninv_ret = ninv
            ninv_cho = np.sqrt(ninv)
        else:
            ninv_cho = np.zeros_like(ninv)
            for drind in range(self.num_draw):
                ninv_cho[drind] = cholesky(ninv[drind], lower=True)
            ninv_ret = ninv

        return ninv_ret, ninv_cho

    def get_sys_cov_samp(self, sys_current, num_class, aff_mode):
        """
        Get a covariance or variance sample based on the current systematic
        realizations and which data are considered as contaminated.

        Parameters:
            sys_current (array_like):
                The current systematic realizations.

            num_class (int):
                Number of data vectors falling into whichver class is being sampled.

            aff_mode (bool):
                Whether sampling from "affected" class (True) or "unaffected"
                class (False).

        Returns:
            sys_cov_samp (array_like or float):
                A covariance or variance sample depending on which class is
                being sampled.
        """

        if aff_mode:
            df = self.df_prior_aff
            scale = self.scale_prior_aff
            if sys_current is not None:
                scatmat = np.einsum('ij,ik->jk', sys_current, sys_current, optimize=True)
            else:
                scatmat = 0
            sys_cov_samp = invwishart(df=df + num_class, scale=scale + scatmat).rvs()
        else:
            df = self.df_prior_unaff
            scale = self.scale_prior_unaff
            if sys_current is not None:
                sum_squares = np.sum(sys_current**2)
            else:
                sum_squares = 0
            sys_cov_samp = invgamma(df + num_class*self.dat_dim / 2, scale=sum_squares/2 + scale).rvs()

        return sys_cov_samp

    def get_sys_samp(self, sys_cov_current, data_use, ninv_use, ninv_cho_use):
        """
        Get the systematic realizations based on the current class labels and
        systematic covariances.

        Parameters:
            sys_cov_current (array or int):
                Current sysematic covariance or variance sample.
            data_use (array):
                data being used for samples
            ninv_use (array):
                Associated inverse noise for data being
                used
            ninv_cho_use (array):
                Associated inverse noise cholesky decomposition for data being
                used

        Returns:
            sys_samp (array): Systematic realization samples.
        """
        num_data_use = data_use.shape[0]
        sys_cov_cho = cholesky(sys_cov_current, lower=True)
        flx = np.random.normal(size=(2, num_data_use, self.dat_dim))

        if self.diag_noise:
            cov_ninv = sys_cov_current * ninv_use[:, np.newaxis] # num_draw, dat_dim, dat_dim
            rhs_flx0 = (sys_cov_current * (ninv_cho_use * flx[0])[:, np.newaxis]).sum(axis=2)
        else:
            cov_ninv = sys_cov_current @ ninv_use # num_draw, dat_dim, dat_dim
            cho_term = (ninv_cho_use * flx[0][:, np.newaxis]).sum(axis=2) # num_draw, dat_dim
            rhs_flx0 = (sys_cov_current[np.newaxis] * cho_term[:, np.newaxis]).sum(axis=2)

        rhs_dat = (cov_ninv * data_use[:, np.newaxis]).sum(axis=2)
        rhs_flx1 = (sys_cov_cho * flx[1][:, np.newaxis]).sum(axis=2)

        rhs_vec = rhs_dat + rhs_flx0 + rhs_flx1
        lhs_op = cov_ninv + np.eye(self.dat_dim) # num_draw, dat_dim, dat_dim

        sys_samp = np.linalg.solve(lhs_op, rhs_vec)

        return sys_samp

    def get_aff_current(self, sys_current, Caff_current, var_current):
        """
        Draw a sample from the distribution of possible classification realizations
        based on the current systematic realizations and covariances/variance samples.

        Parameters:
            sys_current (array): Current systematic realizations
            Caff_current (array): Current sample for systematic covariance
            var_current (array): Current variance sample for unaffected data.

        Returns:
            aff_samp (array):
                Array of bools indicating which data are classified as
                contaminated (True) or uncontaminated (False) for the current
                sample.
        """

        loglikes_aff = multivariate_normal(mean=np.zeros(self.dat_dim), cov=Caff_current).logpdf(sys_current)
        loglikes_unaff = multivariate_normal(mean=np.zeros(self.dat_dim), cov=var_current * np.eye(self.dat_dim)).logpdf(sys_current)

        # FIXME: Needs prior prob if not flat prior!
        log_odds_aff = loglikes_aff - loglikes_unaff
        # this is odds_aff / (odds_aff + 1)
        p_affs = 1 / (np.exp(-log_odds_aff) + 1)

        aff_samp = (np.random.uniform(size=self.num_draw) < p_affs)

        return aff_samp

    def get_post_samps(self, s0, Caff0, var0, aff0, Niter=int(1e5), do_affsamps=False):
        """
        Get samples from the joint posterior of the systematic realizations,
        systematic covariance, systematic variance for unaffected data
        (should always be nearly 0), and classification realizations using Gibbs
        sampling.

        Parameters:
            s0 (array):
                Initial sample for systematic realizations in each data vector.
            Caff0 (array):
                Initial sample for systematic covariance
            Niter (int):
                Number of joint samples to draw
            do_affsamps (bool):
                Whether to classify the data into affected and unaffected classes.
                True means classify, else assume every data vector is affected
                and skip classification steps (much faster, but inaccurate covariance
                estimation when only some of the data is contaminated).

        Returns:
            sys_samps (array):
                Systematic realization samples
            Caff_samps (array):
                Systematic covariance realization samples
            var_samps (array):
                Samples for systematic variance of unaffected data. This is
                basically a dummy variable since in theory it should always be
                zero, however using a delta-function prior produces numerical
                instabilities.
            aff_samps (array):
                Samples of which data vectors are considered as contaminated in
                each iteration.
        """
        sys_samps = np.zeros((Niter, self.num_draw, self.dat_dim))
        Caff_samps = np.zeros((Niter, self.dat_dim, self.dat_dim))
        var_samps = np.zeros(Niter)
        aff_samps = np.zeros((Niter, self.num_draw))

        Caff_current = Caff0
        var_current = var0
        num_sys = self.num_draw
        sys_current = s0
        aff_current = aff0
        for iter_ind in range(Niter):
            if do_affsamps:

                not_aff_current = np.logical_not(aff_current)
                num_aff_current = aff_current.sum()
                # Update sys
                if np.any(aff_current):
                    sys_current[aff_current] = self.get_sys_samp(Caff_current,
                                                                 self.data[aff_current],
                                                                 self.ninv[aff_current],
                                                                 self.ninv_cho[aff_current])
                    # Update covariances
                    Caff_current = self.get_sys_cov_samp(sys_current[aff_current], num_aff_current, aff_mode=True)
                else:
                    # pull from prior
                    Caff_current = self.get_sys_cov_samp(None, num_aff_current, aff_mode=True)

                if np.any(not_aff_current):
                    sys_current[not_aff_current] = self.get_sys_samp(var_current * np.eye(self.dat_dim),
                                                                     self.data[not_aff_current],
                                                                     self.ninv[not_aff_current],
                                                                     self.ninv_cho[not_aff_current])
                    var_current = self.get_sys_cov_samp(sys_current[not_aff_current],
                                                        self.num_draw - num_aff_current,
                                                        aff_mode=False)
                else:
                    var_current = self.get_sys_cov_samp(None,
                                                        self.num_draw - num_aff_current,
                                                        aff_mode=False)
                sys_samps[iter_ind] = sys_current
                Caff_samps[iter_ind] = Caff_current
                var_samps[iter_ind] = var_current

                # Update affected
                aff_current = self.get_aff_current(sys_current, Caff_current, var_current)
                aff_samps[iter_ind] = aff_current

            else:
                sys_current = self.get_sys_samp(Caff_current, self.data)
                sys_samps[iter_ind] = sys_current

                Caff_current = self.get_sys_cov_samp(sys_current, self.num_draw, aff_mode=True)
                Caff_samps[iter_ind] = Caff_current

        return sys_samps, Caff_samps, var_samps, aff_samps


import sys
import random
import numpy as np
from apex import amp
from model import DBGB

from sklearn.model_selection import train_test_split

from transformers import AdamW
import torch
from scipy.sparse import load_npz
from torch.utils.data import DataLoader
from dataset import MDataset, createDataCSV
from log import Logger

import warnings
warnings.filterwarnings("ignore")
def train(model, df, label_map):
    tokenizer = model.get_tokenizer()

    train_d = MDataset(df, 'train', tokenizer, label_map, args.max_len)
    test_d = MDataset(df, 'test', tokenizer, label_map, args.max_len)
    trainloader = DataLoader(train_d, batch_size=args.batch, num_workers=2,shuffle=True)
    testloader = DataLoader(test_d, batch_size=args.batch, num_workers=1,shuffle=False)
    model.cuda()
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.lr)#, eps=1e-8)

    model, optimizer = amp.initialize(model, optimizer, opt_level="O1")

    max_only_p5 = 0
    for epoch in range(0, args.epoch):
        train_loss = model.one_epoch(epoch, trainloader, optimizer, mode='train',
                                     eval_loader=validloader if args.valid else testloader,
                                     eval_step=args.eval_step, log=LOG)
        if args.valid:
            ev_result = model.one_epoch(epoch, validloader, optimizer, mode='eval')
        else:
            ev_result = model.one_epoch(epoch, testloader, optimizer, mode='eval')

        g_p1, g_p3, g_p5, p1, p3, p5,dcg3, dcg5, psp1,psp3,psp5= ev_result
        log_str = f'{epoch:>2}: {p1:.4f}, {p3:.4f}, {p5:.4f}, dcg3: {dcg3:.4f}, dcg5: {dcg5:.4f}, psp1: {psp1:.4f},psp3: {psp3:.4f}, psp5: {psp5:.4f}, train_loss:{train_loss}'

        if args.valid:
            log_str += ' valid'
        LOG.log(log_str)

        if max_only_p5 < p5:
            max_only_p5 = p5
            model.save_model(f'models/model-{get_exp_name()}.bin')

def get_exp_name():
    name = [args.dataset, '' if args.bert == 'bert-base' else args.bert]
    name.append(str(args.maskNum))
    name.append(str(args.headtotail))
    name.append(str(args.Gscale))
    name.append(str(args.scale))
    name.append('tmp')
    return '_'.join([i for i in name if i != ''])

def init_seed(seed):
    np.random.seed(seed)
    random.seed(seed)
    torch.manual_seed(seed)

import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--batch', type=int, required=False, default=16)
parser.add_argument('--update_count', type=int, required=False, default=1)
parser.add_argument('--lr', type=float, required=False, default=0.0001)
parser.add_argument('--seed', type=int, required=False, default=6088)#6088
parser.add_argument('--epoch', type=int, required=False, default=20)
parser.add_argument('--dataset', type=str, required=False, default='eurlex4k')
parser.add_argument('--bert', type=str, required=False, default='bert-base')

parser.add_argument('--max_len', type=int, required=False, default=512)

parser.add_argument('--valid', action='store_true')

parser.add_argument('--swa', action='store_true')
parser.add_argument('--swa_warmup', type=int, required=False, default=10)
parser.add_argument('--swa_step', type=int, required=False, default=100)
parser.add_argument('--eval_step', type=int, required=False, default=20000)
parser.add_argument('--hidden_dim', type=int, required=False, default=300)
parser.add_argument('--eval_model', action='store_true')
parser.add_argument('--scale', type=float, required=False, default=0.5)
parser.add_argument('--maskNum', type=float, required=False, default=0.2)
parser.add_argument('--headtotail', type=float, required=False, default=0.2)
parser.add_argument('--Gscale', type=float, required=False, default=2)

args = parser.parse_args()

if __name__ == '__main__':
    init_seed(args.seed)

    print(get_exp_name())

    LOG = Logger('log_'+get_exp_name())

    print(f'load {args.dataset} dataset...')
    name_map = {'wiki31k': 'Wiki10-31K',
                'amazoncat13k': 'AmazonCat-13K',
                'eurlex4k': 'Eurlex-4K'}

    df, label_map, freq ,train_num= createDataCSV(args.dataset)
    if args.valid:
        train_df, valid_df = train_test_split(df[df['dataType'] == 'train'],
                                              test_size=4000,
                                              random_state=1240)
        df.iloc[valid_df.index.values, 2] = 'valid'
        print('valid size', len(df[df['dataType'] == 'valid']))

    print(f'load {args.dataset} dataset with '
          f'{len(df[df.dataType =="train"])} train {len(df[df.dataType =="test"])} test with {len(label_map)} labels done')

    model = DBGB(n_labels=len(label_map), bert=args.bert, scale = args.scale,maskNum = args.maskNum,
                         update_count=args.update_count,freq=freq, headtotail=args.headtotail, use_swa=args.swa, swa_warmup_epoch=args.swa_warmup,train_num=train_num, swa_update_step=args.swa_step,Gscale=args.Gscale,maxepoch=args.epoch)
    train(model, df, label_map)


from django.contrib import admin
from django.urls import path, include
from . import views

# Defines URL patterns for the Django application.

urlpatterns = [
    # Includes URLs from the 'users' app.
    path('', include('users.urls')),

    # Maps the admin URL to the Django admin site.
    path('admin/', admin.site.urls),

    # Maps the root URL to the blog_index view.
    path("", views.blog_index, name="blog_index"),

    # Maps URLs with the pattern 'post/<int:pk>/' to the blog_detail view.
    # '<int:pk>' is a placeholder for the primary key of a post.
    path("post/<int:pk>/", views.blog_detail, name="blog_detail"),

    # Maps URLs with the pattern 'category/<category>/' to the blog_category view.
    # '<category>' is a placeholder for the category name.
    path("category/<category>/", views.blog_category, name="blog_category"),
]


import typing
import pandas as pd
from data.provider import DataProvider
from data.raw import RawData
from summarize.figures.artists_bar_chart import artists_bar_chart
from summarize.figures.genres_bar_chart import genres_bar_chart
from summarize.figures.labels_bar_chart import labels_bar_chart
from summarize.figures.producers_bar_chart import producers_bar_chart
from summarize.figures.top_artists_time_series import top_artists_time_series
from summarize.figures.top_tracks_time_series import top_tracks_time_series
from summarize.pages.track_features import make_track_features_page
from summarize.pages.clusters import make_clusters_page
from summarize.tables.artists_table import artists_table
from summarize.tables.genres_table import genres_table
from summarize.tables.labels_table import labels_table
from summarize.tables.playlists_table import playlists_table
from summarize.tables.producers_table import producers_table
from summarize.tables.top_artists_table import top_artists_table
from summarize.tables.top_tracks_table import top_tracks_table
from utils.top_lists import get_term_length_phrase, top_list_terms
from utils.track_features import comparison_scatter_plot
from utils.markdown import md_link, md_summary_details, md_truncated_table
from utils.path import errors_path, overview_artist_graph_path, overview_artists_scatterplot_path, overview_audio_features_figure_path, overview_audio_features_path, overview_clusters_figure_path, overview_clusters_path, overview_genre_graph_path, overview_genres_scatterplot_path, overview_label_graph_path, overview_playlists_scatterplot_path, overview_path, overview_producers_graph_path, overview_top_artists_time_series_path, overview_top_tracks_time_series_path
from utils.settings import output_dir

def make_overview(tracks_full: pd.DataFrame):
    print("Generating Overview")

    content = []

    content += title("jbrown1618")
    content += byline()
    content += [md_link(f"See Track Features", overview_audio_features_path(output_dir())), ""]
    content += [md_link(f"See Clusters", overview_clusters_path(output_dir())), ""]
    content += playlists_section()
    content += artists_section(tracks_full)
    content += tracks_section(tracks_full)
    content += genres_section(tracks_full)
    content += labels_section(tracks_full)
    content += producers_section(tracks_full)
    content += errors()

    with open(overview_path(), "w") as f:
        f.write("\n".join(content))

    make_track_features_page(tracks_full, "All Tracks", overview_audio_features_path(), overview_audio_features_figure_path())
    make_clusters_page(tracks_full, "All tracks", overview_clusters_path(), overview_clusters_figure_path())

def title(user: str):
    return [f"# Spotify Summary for {user}", ""]

def byline():
    return [f"Generated by {md_link('jbrown1618/spotify-stats', 'https://github.com/jbrown1618/spotify-stats')}", ""]

def errors():
    return ['## Possible organizational errors', md_link("Possible organizational errors", errors_path(output_dir()))]

def tracks_section(tracks: pd.DataFrame):
    images = [
        md_summary_details(
            f'Top tracks of {get_term_length_phrase(term)} over time',
            top_tracks_time_series(
                term,
                10,
                overview_top_tracks_time_series_path(term),
                overview_top_tracks_time_series_path(term, output_dir())
            )
        )
        for term in top_list_terms
    ]

    contents = [
        "## Tracks",
        "",
        "Top tracks of the last month, six months, and all time",
        "",
        md_truncated_table(top_tracks_table(tracks), 10, "See top 50 tracks"),
        ""
    ]

    return contents + images

def playlists_section():
    dp = DataProvider()
    table = md_truncated_table(playlists_table(output_dir()), 10)

    playlists_sorted_by_track_count = dp.playlists().sort_values(by="playlist_track_count", ascending=False)["playlist_uri"]

    main_playlist_col = dp.tracks()["track_uri"]\
        .apply(lambda track_uri: get_main_playlist(track_uri, playlists_sorted_by_track_count))

    scatter = comparison_scatter_plot(dp.tracks(), main_playlist_col, "Playlist", overview_playlists_scatterplot_path(), overview_playlists_scatterplot_path(output_dir()))

    return [
        "## Playlists",
        "",
        table,
        "",
        scatter,
        ""
    ]

def artists_section(tracks: pd.DataFrame):
    time_series_images = [
        md_summary_details(
            f'Top artists of {get_term_length_phrase(term)} over time',
            top_artists_time_series(
                term,
                10,
                overview_top_artists_time_series_path(term),
                overview_top_artists_time_series_path(term, output_dir())
            )
        )
        for term in top_list_terms
    ]

    top_artists = md_truncated_table(top_artists_table(), 10, "See top 50 artists")

    bar_chart = artists_bar_chart(tracks, overview_artist_graph_path(), overview_artist_graph_path(output_dir()))
    table_data = artists_table(tracks, output_dir())

    summary = f"See all {len(table_data)} artists"
    if len(table_data) > 100:
        summary = "See top 100 artists"
        table_data = table_data.head(100)

    full_list = md_truncated_table(table_data, 10, summary)

    if len(table_data) >= 10:
        scatterplot = comparison_scatter_plot(
            tracks,
            tracks["primary_artist_name"],
            "Artist",
            overview_artists_scatterplot_path(),
            overview_artists_scatterplot_path(output_dir())
        )
    else:
        scatterplot = ""

    return [
        "## Artists",
        "",
        "Top artists of the last month, six months, and all time",
        "",
        top_artists,
        ""
    ] + time_series_images + [
        "",
        "Artists by number of liked tracks",
        full_list,
        "",
        bar_chart,
        "",
        scatterplot,
        ""
    ]

def genres_section(tracks: pd.DataFrame):
    img = genres_bar_chart(tracks, overview_genre_graph_path(), overview_genre_graph_path(output_dir()))
    main_genre_col = tracks["track_uri"].apply(get_main_genre)
    scatter = comparison_scatter_plot(tracks, main_genre_col, "Genre", overview_genres_scatterplot_path(), overview_genres_scatterplot_path(output_dir()))

    table_data = genres_table(tracks, output_dir())

    summary = f"See all {len(table_data)} genres"
    if len(table_data) > 100:
        summary = "See top 100 genres"
        table_data = table_data.head(100)

    full_list = md_truncated_table(table_data, 10, summary)

    return ["## Genres", "", full_list, "", img, "", scatter, ""]

def get_main_genre(track_uri: str):
    track_genre = DataProvider().track_genre()
    subset = track_genre[track_genre["track_uri"] == track_uri]
    if len(subset) == 0:
        return "None"

    return subset.iloc[0]["genre"]

def get_main_playlist(track_uri: str, playlists_sorted_by_track_count: typing.Iterable[str]):
    playlists = DataProvider().playlists()
    playlist_track = RawData()['playlist_track']

    playlist_uris = set(playlist_track[playlist_track["track_uri"] == track_uri]["playlist_uri"])
    for playlist_uri in playlists_sorted_by_track_count:
        if playlist_uri in playlist_uris:
            return playlists[playlists["playlist_uri"] == playlist_uri].iloc[0]["playlist_name"]
    return "None"

def labels_section(tracks: pd.DataFrame):
    img = labels_bar_chart(tracks, overview_label_graph_path(), overview_label_graph_path(output_dir()))
    table_data = labels_table(tracks, output_dir())

    summary = f"See all {len(table_data)} labels"
    if len(table_data) > 100:
        summary = "See top 100 labels"
        table_data = table_data.head(100)

    full_list = md_truncated_table(table_data, 10, summary)

    return ["## Record Labels", "", full_list, "", img, ""]

def producers_section(tracks: pd.DataFrame):
    producers = producers_table(tracks, output_dir())

    if len(producers) == 0:
        return []

    bar_chart = producers_bar_chart(
        tracks,
        overview_producers_graph_path(),
        overview_producers_graph_path(output_dir()))

    return [
        '## Top Producers',
        '',
        md_truncated_table(producers),
        '',
        bar_chart
    ]


import numpy as np
import configparser as cp
from network.data_features_dumper import DataFeaturesDumper
from network.silknet import LoadInterface
from network.silknet.FolderDataReader import FolderDataReader
from interface import implements
import os
import pickle
from network.computation_graph_parse import ComputationGraphTableParse
import torch
from torch.autograd import Variable
import cv2

class DataLoader(implements(LoadInterface)):
    def load_datum(self, full_path):
        with open(os.path.join(full_path, '__dump__.pickle'), 'rb') as f:
            doc = pickle.load(f)
        return doc

class Trainer:
    def __init__(self):
        config = cp.ConfigParser()
        config.read('config.ini')
        self.train_path = config['parse']['train_data_path']
        self.test_path = config['parse']['test_data_path']
        self.validation_data_path = config['parse']['validation_data_path']
        self.glove_path = config['parse']['glove_path']
        self.learning_rate = float(config['parse']['learning_rate'])
        self.from_scratch = int(config['parse']['from_scratch']) == 1
        self.model_path = config['parse']['model_path']
        self.save_after = int(config['parse']['save_after'])

    def init(self, dump_features_again):
        pass

    def do_plot(self, document, id):
        rects = document.rects
        row_share = document.row_share
        canvas = (np.ones((500,500, 3))*255).astype(np.uint8)
        for i in range(len(rects)):
            rect = rects[i]
            color = (255, 0, 0) if document.cell_share[0, i] == 0 else (0,0,255)
            cv2.rectangle(canvas, (int(rect[0] * 500), int(rect[1]*500)), (int((rect[0]+rect[2]) * 500), int((rect[1]+rect[3])*500)), color)
        cv2.imshow('test' + id, canvas)
        cv2.waitKey(0)

    def sample_indices(self, vector):
        A = vector.copy()
        num_ones = np.sum(A == 1)

        sample_from = np.where(A == 0)[0]
        np.random.shuffle(sample_from)
        picked_zeros = sample_from[0:min(num_ones, np.size(sample_from))]
        A[picked_zeros] = 1

        return np.where(A == 1)[0]

    def get_example_elements(self, document, id):
        num_words, _ = np.shape(document.rects)
        vv = np.concatenate([document.rects, document.distances, document.embeddings], axis=1).astype(np.float32)
        vv = Variable(torch.from_numpy(vv)).cuda()

        row_share_flatten = document.row_share.flatten()
        col_share_flatten = document.col_share.flatten()
        cell_share_flatten = document.cell_share.flatten()

        row_check_indices = self.sample_indices(row_share_flatten)
        col_check_indices = self.sample_indices(col_share_flatten)
        cell_check_indices = self.sample_indices(cell_share_flatten)

        y_row = row_share_flatten[row_check_indices]
        y_col = row_share_flatten[col_check_indices]
        y_cell = row_share_flatten[cell_check_indices]

        # y_row = Variable(torch.from_numpy(document.row_share.flatten()), requires_grad=False).cuda()
        # y_col = Variable(torch.from_numpy(document.col_share.flatten().astype(np.int64)), requires_grad=False).cuda()
        # y_cell = Variable(torch.from_numpy(document.cell_share.flatten().astype(np.int64)), requires_grad=False).cuda()

        baseline_accuracy_row = 100 * np.sum(y_row == 0) / (num_words * num_words)
        baseline_accuracy_col = 100 * np.sum(y_col == 1) / (num_words * num_words)
        baseline_accuracy_cell = 100 * np.sum(y_cell == 1) / (num_words * num_words)

        y_row = Variable(torch.from_numpy(y_row.astype(np.int64))).cuda()
        y_col = Variable(torch.from_numpy(y_col.astype(np.int64))).cuda()
        y_cell = Variable(torch.from_numpy(y_cell.astype(np.int64))).cuda()

        row_check_indices = torch.from_numpy(row_check_indices).cuda()
        col_check_indices = torch.from_numpy(col_check_indices).cuda()
        cell_check_indices = torch.from_numpy(cell_check_indices).cuda()

        indices = torch.LongTensor(torch.from_numpy(np.concatenate(
            [np.expand_dims(np.arange(num_words, dtype=np.int64), axis=1),
             np.maximum(document.neighbor_graph.astype(np.int64), 0)], axis=1))).cuda()
        indices_not_found = torch.ByteTensor(torch.from_numpy(np.repeat(np.concatenate(
            [np.expand_dims(np.zeros(num_words, dtype=np.int64), axis=1),
             document.neighbor_graph.astype(np.int64)], axis=1) == -1, 100).reshape((-1, 500)).astype(
            np.uint8))).cuda()
        indices_not_found = indices_not_found * 0

        return num_words, vv, 0, 0, indices, indices_not_found, y_row, y_col, y_cell, row_check_indices, col_check_indices, cell_check_indices, baseline_accuracy_row, baseline_accuracy_col, baseline_accuracy_cell

    def do_validation(self, model, dataset):

        sum_of_accuracies = 0
        total = 0
        while True:
            document, epoch, id = dataset.next_element()
            num_words, vv, y, baseline_accuracy_1, baseline_accuracy_2, indices, indices_not_found = self.get_example_elements(document)

            y_pred = model(indices, indices_not_found, vv, num_words)
            _, predicted = torch.max(y_pred.data, 1)

            accuracy = torch.sum(predicted == y.data)
            accuracy = 100 * accuracy / num_words

            print(accuracy)

            total += 1
            sum_of_accuracies += accuracy

            if epoch == 1:
                break

        print("Average validation accuracy = ", sum_of_accuracies / total)

    def train(self):
        dataset = FolderDataReader(self.train_path, DataLoader())
        validation_dataset = FolderDataReader(self.validation_data_path, DataLoader())
        dataset.init()
        validation_dataset.init()
        model = ComputationGraphTableParse()

        if not self.from_scratch:
            model.load_state_dict(torch.load(self.model_path))

        # class_weights = torch.from_numpy(np.array([0.01,0.99]).astype(np.float32)).cuda()

        model.set_iterations(2)
        criterion = torch.nn.CrossEntropyLoss(size_average=True)
        optimizer = torch.optim.Adam(model.parameters(), lr=self.learning_rate)
        iterations = 0

        for i in range(1000000):
            # if i % 10000 == 0:
            #     self.do_validation(model, validation_dataset)

            document, epoch, id = dataset.next_element()
            num_words, vv, baseline_accuracy_1, baseline_accuracy_2, indices, indices_not_found, y_row, y_col, y_cell, row_check_indices, col_check_indices, cell_check_indices, baseline_accuracy_row, baseline_accuracy_col, baseline_accuracy_cell = self.get_example_elements(
                document, id)

            # Save model
            if (iterations % self.save_after) == 0:
                print("Saving model")
                torch.save(model.state_dict(), self.model_path)

            for j in range(1):
                iterations += 1

                row_logits, col_logits, cell_logits = model(indices, indices_not_found, vv, num_words)
                row_logits = row_logits[row_check_indices]
                col_logits = col_logits[col_check_indices]
                cell_logits = cell_logits[cell_check_indices]

                _, predicted = torch.max(row_logits.data, 1)
                accuracy_row = torch.sum(predicted == y_row.data)
                accuracy_row = 100 * accuracy_row / (num_words * num_words)

                _, predicted = torch.max(col_logits.data, 1)
                accuracy_col = torch.sum(predicted == y_col.data)
                accuracy_col = 100 * accuracy_col / (num_words * num_words)

                _, predicted = torch.max(cell_logits.data, 1)
                accuracy_cell = torch.sum(predicted == y_cell.data)
                accuracy_cell = 100 * accuracy_cell / (num_words * num_words)

                loss_row = criterion(row_logits, y_row)
                # loss_col = criterion(col_logits, y_col)
                # loss_cell = criterion(cell_logits, y_cell)
                loss = loss_row #+ loss_col + loss_cell

                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
                print("%3dx%3d Loss = %f" % (i, j, loss.data[0]), "Row: %03.2f" % accuracy_row,
                      "BRow: %03.2f" % baseline_accuracy_row, "Col: %03.2f" % accuracy_col, "BCol: %03.2f" % baseline_accuracy_col,
                      "Cell: %03.2f" % accuracy_cell, "BCell: %03.2f" % baseline_accuracy_cell)


import matplotlib.pyplot as plt
import pandas as pd
from cdrift import evaluation
from cdrift.utils.helpers import readCSV_Lists, convertToTimedelta, importLog
import numpy as np
from datetime import datetime
from statistics import mean, harmonic_mean, stdev
from scipy.stats import iqr
from typing import List, Tuple
import seaborn as sns
import re
import os
from pathlib import Path
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import warnings
from tqdm.auto import tqdm

plt.rcParams['pdf.fonttype'] = 42
plt.rcParams['ps.fonttype'] = 42

mapping_ostovar_to_shortnames = {
    "ConditionalMove": 'cm',
    "ConditionalRemoval": 'cre',
    "ConditionalToSequence": 'cf',
    "Frequency": 'fr',
    "Loop": 'lp',
    "ParallelMove": 'pm',
    "ParallelRemoval": 'pre',
    "ParallelToSequence": 'pl',
    "SerialMove": 'sm',
    "SerialRemoval": 'sre',
    "Skip": 'cb',
    "Substitute": 'rp',
    "Swap": 'sw',
}

shorter_names = {
    "Zheng DBSCAN": "RINV",
    "ProDrift": "ProDrift",
    "Maaradji Runs": "ProDrift",
    "Bose J": "J-Measure",
    "Bose WC": "Window Count",
    "Earth Mover's Distance": "EMD",
    "Process Graph Metrics": "PGM",
    "Martjushev ADWIN J": "ADWIN J",
    "Martjushev ADWIN WC": "ADWIN WC",
    "LCDD": "LCDD"
}

used_parameters = {
    "Bose J": ["Window Size", "SW Step Size"],
    "Bose WC": ["Window Size", "SW Step Size"],
    "Martjushev ADWIN J": ["Min Adaptive Window", "Max Adaptive Window", "P-Value", "ADWIN Step Size"],
    "Martjushev ADWIN WC": ["Min Adaptive Window", "Max Adaptive Window", "P-Value", "ADWIN Step Size"],
    "ProDrift": ["Window Size", "SW Step Size"],
    "Earth Mover's Distance": ["Window Size", "SW Step Size"],
    "Process Graph Metrics": ["Min Adaptive Window", "Max Adaptive Window", "P-Value"],
    "Zheng DBSCAN": ["MRID", "Epsilon"],
    "LCDD": ["Complete-Window Size", "Detection-Window Size", "Stable Period"]
}

used_parameters = {
    shorter_names[name]: used_parameters[name]
    for name in used_parameters.keys()
}

def split_by_name(df):
    return [
        (alg, df[df["Algorithm"] == alg].copy())
        for alg in df["Algorithm"].unique()
    ]

def map_row_to_cp(row):
    logname = row["Log"]
    if row["Log Source"] == "Ceravolo":
        return logname.split("_")[-1]
    elif row["Log Source"] == "Ostovar":
        change_pattern = logname.split("_")[-1]
        if change_pattern.isnumeric(): # This log has a noise level; Use the second to last string instead
            change_pattern = logname.split("_")[-2]
        return mapping_ostovar_to_shortnames[change_pattern]
    elif row["Log Source"] == "Bose":
        return None
    else:
        raise ValueError(f"Unknown Log Source for Versatility: {row['Log Source']}; Is this Log meant to be used for a Versatility Evaluation?")

def map_row_to_noise_level(row):
    logname = row["Log"]
    if row["Log Source"] == "Ceravolo":
        return re.search('noise([0-9]*)_', logname).group(1)
    elif row["Log Source"] == "Ostovar":
        # return mapping_ostovar_to_shortnames[logname.split("_")[-1]]
        last_member = logname.split('_')[-1]
        if last_member.isnumeric():
            return last_member
        else:
            return '0'
    elif row["Log Source"] == "Bose":
        return None
    else:
        raise ValueError(f"Unknown Log Source for Noise Level: {row['Log Source']}; Is this Log meant to be used for a Robustness Evaluation?")

def preprocess(_df):
    df = _df.copy()
    ## Exclude Martjushev WC (in favor of LCDD)
    df = df[df["Algorithm"] != "Martjushev ADWIN WC"]

    ## Rename Approaches to shorter names
    df["Algorithm"] = df["Algorithm"].map(lambda x: shorter_names.get(x, x))

    ## Only use Atomic Logs
    atomic_logs = {
        log
        for _, (source, log) in df[["Log Source", "Log"]].iterrows()
        if (source.lower() == "ostovar" and log.lower().startswith("atomic")) # Only use atomic logs from ostovar
        or (log.lower().startswith("bose")) # Use bose log
        or (source.lower() == "ceravolo" and set(log.lower().split("_")[-1]) != {'i', 'o', 'r'}) # Exclude all composite (IOR, ROI, etc.) logs from ceravolo
    }
    df = df[df["Log"].isin(atomic_logs)]

    # Add a column stating the change pattern that is applied and one for the noise level
    # Only use the Ceravolo and Ostovar Logs as Bose has only 1 log, i.e., no noise levels
    df["Change Pattern"] = df.apply(lambda x: map_row_to_cp(x), axis=1)
    df["Noise Level"] = df.apply(lambda x: map_row_to_noise_level(x), axis=1)

    return df

def evaluate(csv_path="algorithm_results.csv", out_path="Evaluation_Results", lag_window:int=200, min_support:int=1, verbose:bool = False):
    """Evaluate the results of the algorithms.

    Args:
        csv_path (str, optional): Path to results csv file. Defaults to "algorithm_results.csv".
        out_path (str, optional): Directory in which to save results. Defaults to "Evaluation_Results".
        lag_window (int, optional): Max acceptable distance to a ground truth change point to be classified as true positive. Defaults to 200.
        min_support (int, optional): Minimum support for latency calculation --> a parameter setting must have at least 3 instances where a true positive was detected before it can be deemed the "best parameter setting" for latency calculation. Defaults to 1.
        verbose (bool, optional): Whether to print information about the data. Defaults to False.

    Raises:
        ValueError: CSV Contains duplicate rows
    """

    if not os.path.exists(out_path):
        os.makedirs(out_path)

    # Preprocess
    df = readCSV_Lists(csv_path)
    df = preprocess(df)
    ALPHA_SORT_NAMES = sorted(list(df["Algorithm"].unique()))

    ## Calculate Log Lengths for Scalability
    _groupkeys = list(df.groupby(["Log Source", "Log"]).groups.keys())
    _logpaths = [Path("EvaluationLogs", source, f"{log}.xes.gz") for source, log in _groupkeys]

    loglengths = dict()
    loglengths_events = dict()

    for logpath in tqdm(_logpaths, "Calculating Log Lengths. Completed Logs: "):
        log = importLog(logpath.as_posix(), verbose=False)
        loglengths[logpath] = len(log)
        loglengths_events[logpath] = len([evt for case in log for evt in case])

    if verbose:
        print(f"Number of Logs: {len(loglengths.keys())}")
        print(f"Number of Cases: {sum(loglengths.values())}")
        print(f"Number of Events: {sum(loglengths_events.values())}")

        num_drifts = sum(
            len(group.iloc[0]["Actual Changepoints for Log"])
            for name, group in df.groupby(["Log Source", "Log"])
        )
        print("Total number of drifts:", num_drifts)

    dfs = split_by_name(df)

    if pd.read_csv(csv_path).duplicated().any():
        raise ValueError("CSV contains duplicate rows!")

    ## Split into Noisy/Noiseful Logs
    logs = zip(df["Log Source"], df["Log"])
    noiseless_logs = {
        log for source, log in logs if
        (source == "Ostovar"  and not (log.endswith("_2") or log.endswith("_5"))) or
        (source == "Ceravolo" and log.split("_")[2]=="noise0") or
        source == "Bose"
    }

    df_noiseless = df[df["Log"].isin(noiseless_logs)]
    df_noisy = df[df["Log"].isin(noiseless_logs) == False]

    dfs_noisy = split_by_name(df_noisy)
    dfs_noiseless = split_by_name(df_noiseless)

    ## Define Color Scheme
    colors = ["#DB444B","#9A607F","#006BA2","#3EBCD2","#379A8B","#EBB434","#B4BA39","#D1B07C"]
    color_map = {ALPHA_SORT_NAMES[i]:colors[i] for i in range(0,len(ALPHA_SORT_NAMES))}

    analyze_change_pattern_distribution(df, out_path)

    # Accuracy
    accuracies, computed_accuracy_dicts, computed_precision_dicts, computed_recall_dicts, accuracy_best_param = calculate_accuracy_metric_df(df_noiseless, lag_window, show_progress_bar=True)
    plot_accuracy(computed_accuracy_dicts, computed_precision_dicts, computed_recall_dicts, accuracy_best_param, out_path, lag_window, colors, ALPHA_SORT_NAMES)

    # Latency
    latencies, scaled_latency_dicts, computed_latency_dicts, best_params_latency = calculate_latency(df_noiseless, lag_window, min_support=min_support, show_progress_bar=True)
    plot_latency(computed_latency_dicts, best_params_latency, colors, out_path, lag_window, ALPHA_SORT_NAMES)

    # Versatility
    ## Only use Ceravolo and Ostovar Logs because these are the only ones that have different change patterns.
    ## (Theoretically we could also drop all rows containing a nan in the change pattern column? This would be more general)
    df_v = df_noiseless[df_noiseless["Log Source"].isin(["Ceravolo", "Ostovar"])].copy(deep=True)
    versatilities, versatility_recall_dicts, best_params_versatility = calc_versatility(df_v, lag_window, show_progress_bar=True)
    plot_versatility(versatility_recall_dicts, best_params_versatility, out_path, lag_window, colors, ALPHA_SORT_NAMES)

    # Scalability
    scalabilities = calculate_scalability(df, show_progress_bar=True)
    plot_scalability(scalabilities, dfs, out_path, lag_window, colors, ALPHA_SORT_NAMES)
    df_seconds, seconds_per_case, seconds_per_event = calculate_rel_scalabilities(df, loglengths, loglengths_events)
    plot_rel_scalability(df_seconds, out_path, lag_window, colors, ALPHA_SORT_NAMES)

    # Parameter Sensitivity
    sensitivities = calculate_parameter_sensitivity(df, versatility_recall_dicts, computed_accuracy_dicts, scaled_latency_dicts, show_progress_bar=True)
    sensitivity_iqrs = calculate_parameter_sensitivity_iqr(sensitivities)
    plot_parameter_sensitivity(sensitivities, out_path, lag_window, colors, ALPHA_SORT_NAMES)

    # Robustness
    df_robust = df[df["Log Source"].isin(["Ceravolo", "Ostovar"])].copy(deep=True)
    means_ceravolo = calc_harm_means(df_robust[df_robust["Log Source"] == "Ceravolo"], min_support, lag_window, show_progress_bar=True)
    means_ostovar = calc_harm_means(df_robust[df_robust["Log Source"] == "Ostovar"], min_support, lag_window, show_progress_bar=True)
    robustness_ceravolo = convert_harm_mean_to_auc(means_ceravolo)
    robustness_ostovar = convert_harm_mean_to_auc(means_ostovar)
    # Calculate final robustness score by calculating mean of ostovar and ceravolo performance
    robustnesses = {
        name: (robustness_ceravolo[name] + robustness_ostovar[name]) / 2
        for name in robustness_ceravolo.keys()
    }
    plot_robustness(means_ostovar, means_ceravolo, out_path, lag_window, color_map)

    # Save results to csv
    results_df = pd.DataFrame([
        {
            "Algorithm": name,
            "Accuracy": accuracies[name],
            "Latency": (1-latencies[name])*lag_window,
            "Versatility": versatilities[name],
            "Scalability": scalabilities[name]["str"],
            "Milliseconds per Case": seconds_per_case[name]*1000,
            "Milliseconds per Event": seconds_per_event[name]*1000,
            "Parameter Sensitivity (IQR)": sensitivity_iqrs[name],
            "Robustness": robustnesses[name]
        }
        for name in df["Algorithm"].unique()
    ])

    results_df.to_csv(f"{out_path}/evaluation_measures{lag_window}.csv", index=False)

def analyze_change_pattern_distribution(df, out_path):
    cp_count_results = { cp: dict() for cp in df["Change Pattern"].unique() if cp is not None } # Bose gets lost because it maps to none

    for name, group in df[["Log Source", "Log", "Change Pattern"]].drop_duplicates(inplace=False).groupby("Log Source"): # Only consider each logpath once (not once for each algorithm application)
        series = group["Change Pattern"].value_counts().to_dict()
        for cp, count in series.items():
            cp_count_results[cp][name] = count

    for key in cp_count_results.keys():
        cp_count_results[key]["Bose"] = 0
    cp_count_results["Mixed (Bose)"] = {"Ostovar": 0, "Ceravolo": 0, "Bose": 1}

    cp_counts = pd.DataFrame(cp_count_results).fillna(0).astype(int)

    # Analyze relative counts
    cp_counts_tr = cp_counts.transpose()
    total_logs = cp_counts.to_numpy().sum()
    cp_counts_tr["Relative Frequency"] = cp_counts_tr.apply(lambda row: row.sum() / total_logs * 100, axis=1)

    cp_counts_tr.to_csv(f"{out_path}/change_pattern_distribution.csv")

def calcAccuracy(df:pd.DataFrame, param_names:List[str], lag_window: int):
    """Calculates the Accuracy Metric for the given dataframe by grouping by the given parameters and calculating the mean accuracy

    Args:
        df (pd.DataFrame): The dataframe containing the results to be evaluated
        param_names (List[str]): The names of the parameters of this approach
        lag_window (int): The lag window to be used for the evaluation to determine true positives and false positives
    """

    f1s = dict()
    recalls = dict()
    precisions = dict()

    # Group by parameter values to calculate accuracy per parameter setting, over all logs
    for parameters, group in df.groupby(by=param_names):
        # Calculate Accuracy for this parameter setting
        ## --> F1-Score, but first collect all TP and FP
        tps = 0
        fps = 0
        positives = 0
        detected = 0

        # Collect TP FP, etc.
        for index, row in group.iterrows():
            actual_cp = row["Actual Changepoints for Log"]
            detected_cp = row["Detected Changepoints"]
            tp, fp = evaluation.getTP_FP(detected_cp, actual_cp, lag_window)
            tps += tp
            fps += fp
            positives += len(actual_cp)
            detected += len(detected_cp)

        try:
            precisions[parameters] = tps / detected
        except ZeroDivisionError:
            precisions[parameters] = np.NaN

        try:
            recalls[parameters] = tps / positives
        except ZeroDivisionError:
            recalls[parameters] = np.NaN

        f1s[parameters] = harmonic_mean([precisions[parameters], recalls[parameters]]) # If either is nan, the harmonic mean is nan
    return (precisions, recalls, f1s)

def calculate_accuracy_metric_df(dataframe, lag_window, show_progress_bar: bool = False):
    computed_accuracy_dicts = dict()
    computed_precision_dicts = dict()
    computed_recall_dicts = dict()

    accuracy_best_param = dict()

    accuracies = dict()

    groups = dataframe.groupby(by="Algorithm")
    if show_progress_bar:
        groups = tqdm(groups, "Calculating Accuracy. Completed Algorithms: ")
    for name, a_df in groups:
        computed_precision_dicts[name], computed_recall_dicts[name], computed_accuracy_dicts[name] = calcAccuracy(a_df, used_parameters[name], lag_window)

        best_param = max(computed_accuracy_dicts[name],  key=lambda x: computed_accuracy_dicts[name][x])

        accuracy_best_param[name] = best_param

        # accuracies[name] = max(computed_accuracy_dicts[name].values())
        accuracies[name] = computed_accuracy_dicts[name][best_param]
    return (accuracies, computed_accuracy_dicts, computed_precision_dicts, computed_recall_dicts, accuracy_best_param)

def plot_accuracy(computed_accuracy_dicts, computed_precision_dicts, computed_recall_dicts, accuracy_best_param, out_path, lag_window, colors, order):
    accuracy_plot_df = pd.DataFrame(
        [
            {
                "Algorithm": name,
                "Metric": "F1-Score",
                "Value": computed_accuracy_dicts[name][accuracy_best_param[name]]
            }
            for name in computed_accuracy_dicts.keys()
        ] + [
            {
                "Algorithm": name,
                "Metric": "Precision",
                "Value": computed_precision_dicts[name][accuracy_best_param[name]]
            }
            for name in computed_accuracy_dicts.keys()
        ] + [
            {
                "Algorithm": name,
                "Metric": "Recall",
                "Value": computed_recall_dicts[name][accuracy_best_param[name]]
            }
            for name in computed_accuracy_dicts.keys()
        ]
    )

    palette=None #{"Precision": "#573deb", "F1-Score": "#ff0076", "Recall": "#ffa600"}
    palette = sns.color_palette(colors)
    plt.grid(zorder=0)
    ax = sns.barplot(x="Metric", y="Value", data=accuracy_plot_df, hue="Algorithm", palette=palette, hue_order=order,zorder =5)

    ax.figure.set_size_inches(11, 4)
    ax.spines.right.set_visible(False)
    ax.spines.top.set_visible(False)
    ax.tick_params(labelsize=18)
    ax.set_xlabel(ax.get_xlabel(), size = 20)
    ax.set_ylabel("", size = 20)
    #ax.get_yaxis().set_visible(False)
    plt.legend(bbox_to_anchor=(1.01, 1), loc='upper left', borderaxespad=0,prop={'size': 14})

    if not os.path.exists(f"{out_path}/Accuracy{lag_window}"):
        os.makedirs(f"{out_path}/Accuracy{lag_window}")
    plt.savefig(f"{out_path}/Accuracy{lag_window}/accuracy_c.pdf", bbox_inches="tight", format="pdf")

    ax = sns.pointplot(
        x="Algorithm",
        y="Value",
        data=accuracy_plot_df,
        hue="Metric",
        order=order,
        join=True,
    )
    ax.figure.set_size_inches(15, 4)
    plt.savefig(f"{out_path}/Accuracy{lag_window}/accuracy_points.pdf", bbox_inches="tight", format="pdf")

def calcLatencies(df, param_names, lag_window, show_progress_bar: bool = False):
    latencies = dict()
    for parameters, group in df.groupby(by=param_names):
        lags = []
        for index, row in group.iterrows():
            actual_cp = row["Actual Changepoints for Log"]
            detected_cp = row["Detected Changepoints"]
            assignments = evaluation.assign_changepoints(detected_cp, actual_cp, lag_window)
            for d, a in assignments:
                lags.append(abs(d-a))
        latencies[parameters] = lags
    return latencies

def calculate_latency(dataframe, lag_window, min_support=1, show_progress_bar: bool = False):
    latencies = dict() # Dict holding the best achieved latency per approach
    scaled_latency_dicts = dict() # Dict holding the scaled mean latency per approach per parameter setting
    computed_latency_dicts = dict() # Dict holding the raw list of detection lags per approach per parameter setting
    best_params_latency = dict() # Dict holding the best parameter setting per approach (the one that achieves the best latency)

    groups = dataframe.groupby(by="Algorithm")
    if show_progress_bar:
        groups = tqdm(groups, "Calculating Latency. Completed Algorithms: ")
    for name, a_df in groups:
        result = calcLatencies(a_df, used_parameters[name], lag_window)
        computed_latency_dicts[name] = result
        latency_scaled_dict = {
            param: 1-(mean(result[param])/lag_window) if len(result[param]) >= min_support else np.NaN
            for param in result.keys()
        }
        scaled_latency_dicts[name] = latency_scaled_dict
        best_param = max(latency_scaled_dict,  key=lambda x: latency_scaled_dict[x] if not np.isnan(latency_scaled_dict[x]) else -1)

        best_params_latency[name] = best_param
        latencies[name] = latency_scaled_dict[best_param]

    return (latencies, scaled_latency_dicts, computed_latency_dicts, best_params_latency)

def plot_latency(computed_latency_dicts, best_params_latency, colors, out_path, lag_window, order):
    latency_plot_df = pd.DataFrame([
    {
        "Algorithm": name,
        "Unscaled Latency": latency
    }
    for name in computed_latency_dicts.keys()
    for latency in computed_latency_dicts[name][best_params_latency[name]]
    ])

    ax = plt.subplots(figsize=(17, 4))

    palette = sns.color_palette(colors)

    ax = sns.barplot(x="Algorithm", y="Unscaled Latency", data=latency_plot_df,palette= palette, order=order)
    plt.ylabel("Latency")# [Traces]")
    ax.spines.right.set_visible(False)
    ax.spines.top.set_visible(False)
    ax.tick_params(labelsize=16)
    ax.set_xlabel(ax.get_xlabel(), size = 18)
    ax.set_ylabel(ax.get_ylabel(), size = 18)
    ax.figure.set_size_inches(14, 4)

    if not os.path.exists(f"{out_path}/Latency{lag_window}"):
        os.makedirs(f"{out_path}/Latency{lag_window}")
    plt.savefig(f"{out_path}/Latency{lag_window}/latency.pdf", bbox_inches="tight", format="pdf")

def calc_versatility(dataframe, lag_window, show_progress_bar: bool = False):

    versatility_recall_dicts = dict() # Map approach to a dictionary mapping param setting to mean recall over all change patterns
    versatilities = dict() # Map approach to versatility score
    best_params_versatility = dict() # Map approach to best param setting

    groups = dataframe.groupby(by="Algorithm")
    if show_progress_bar:
        groups = tqdm(groups, "Calculating Versatility. Completed Algorithms: ")

    for name, group in groups:
        recalls_of_this_approach = dict()
        for params, params_group in group.groupby(by=used_parameters[name]):
            recalls = dict()
            for change_pattern, cp_group in params_group.groupby(by="Change Pattern"):
                TPS = 0
                POSITIVES = 0
                # TP / TP+FN = TP / POSTIVES = Recall
                for index, row in cp_group.iterrows():
                    detected_changepoints = row["Detected Changepoints"]
                    actual_changepoints = row["Actual Changepoints for Log"]

                    tp, _ = evaluation.getTP_FP(detected_changepoints, actual_changepoints, lag_window)

                    TPS += tp
                    POSITIVES += len(actual_changepoints)

                # Recall of this algorithm for this change pattern:
                recall = TPS / POSITIVES if POSITIVES != 0 else np.NaN # Only the case if there are no actual changepoints, which should not be the case
                recalls[change_pattern] = recall
            recalls_of_this_approach[params] = recalls
        versatility_recall_dicts[name] = recalls_of_this_approach
        best_param = max(recalls_of_this_approach, key=lambda x: np.nanmean(list(recalls_of_this_approach[x].values()))) # .values() gives us all the recalls for all change patterns on this param setting
        best_params_versatility[name] = best_param
        versatilities[name] = np.nanmean(list(
            recalls_of_this_approach[best_param].values()
        ))

    return versatilities, versatility_recall_dicts, best_params_versatility

def plot_versatility(versatility_recall_dicts, best_params_versatility, out_path, lag_window, colors, order):
    df_vers_plot = pd.DataFrame([
        {
            "Algorithm": name,
            "Change Pattern": cp,
            "Versatility": versatility_recall_dicts[name][best_params_versatility[name]][cp]
        }
        for name in versatility_recall_dicts.keys()
        for cp in versatility_recall_dicts[name][best_params_versatility[name]].keys()
    ])

    fig,ax = plt.subplots(figsize=(20, 4))
    sns.barplot(x="Change Pattern", y="Versatility", data=df_vers_plot, hue="Algorithm", ax=ax, hue_order=order)
    plt.legend(bbox_to_anchor=(1.01, 1), loc='upper left', borderaxespad=0)

    if not os.path.exists(f"{out_path}/Versatility{lag_window}"):
        os.makedirs(f"{out_path}/Versatility{lag_window}")
    plt.savefig(f"{out_path}/Versatility{lag_window}/versatility.pdf", bbox_inches="tight", format="pdf")

    plt.close('all')

    # As points
    change_patterns = sorted(list(df_vers_plot["Change Pattern"].unique()))
    ax = sns.pointplot(
        x="Change Pattern",
        y="Versatility",
        data=df_vers_plot,
        hue="Algorithm",
        hue_order=order,
        order=change_patterns
    )
    ax.figure.set_size_inches(20,4)
    plt.legend(bbox_to_anchor=(1.01, 1), loc='upper left', borderaxespad=0)
    plt.savefig(f"{out_path}/Versatility{lag_window}/versatility_line_graph.pdf", bbox_inches="tight", format="pdf")

    # Plot bars again, but split in half
    first_half = change_patterns[:len(change_patterns)//2]
    second_half = change_patterns[len(change_patterns)//2:]
    fig,ax = plt.subplots(figsize=(20, 4))
    sns.barplot(x="Change Pattern", y="Versatility", data=df_vers_plot[df_vers_plot["Change Pattern"].isin(first_half)], hue="Algorithm", order=first_half, ax=ax, hue_order=order)
    plt.legend(bbox_to_anchor=(1.01, 1), loc='upper left', borderaxespad=0)

    plt.savefig(f"{out_path}/Versatility{lag_window}/versatility_split_1.pdf", bbox_inches="tight", format="pdf")

    fig2,ax2 = plt.subplots(figsize=(20, 4))
    sns.barplot(x="Change Pattern", y="Versatility", data=df_vers_plot[df_vers_plot["Change Pattern"].isin(second_half)], hue="Algorithm", order=second_half, ax=ax2, hue_order=order)
    plt.legend(bbox_to_anchor=(1.01, 1), loc='upper left', borderaxespad=0)
    plt.savefig(f"{out_path}/Versatility{lag_window}/versatility_split_2.pdf", bbox_inches="tight", format="pdf")

    # Versatility as Radar Chart
    plotly_default_colors = colors#px.colors.qualitative.Plotly
    fill_colors = {
        name: plotly_default_colors[idx]
        for idx,name in enumerate(df_vers_plot["Algorithm"].unique())
    }

    with warnings.catch_warnings():
        warnings.simplefilter(action='ignore', category=FutureWarning)
        for alg, dataframe in df_vers_plot.groupby(by="Algorithm"):
            fig = px.line_polar(
                df_vers_plot[df_vers_plot["Algorithm"] == alg], # Isnt this just dataframe from the groupby...
                r='Versatility',
                theta='Change Pattern',
                line_close=True,
                color='Algorithm',
                color_discrete_map=fill_colors,
                title=None# alg, # Set this to alg if we want the title, I was planning on setting the title in latex
            )
            fig.update_layout(showlegend=False, title_x=0.5, font=dict(size=18)) # Disable legend (as we have only one algorithm per plot) and center title (if present)
            fig.update_traces(fill='toself')

            if not os.path.exists(f"{out_path}/Versatility{lag_window}/Radar_Charts"):
                os.makedirs(f"{out_path}/Versatility{lag_window}/Radar_Charts")
            fig.write_image(f"{out_path}/Versatility{lag_window}/Radar_Charts/radar_chart_{alg.replace(' ','_')}.pdf", format="pdf")

def calculate_scalability(dataframe, show_progress_bar: bool = False):
    scalabilities = dict()

    groups = dataframe.groupby("Algorithm")
    if show_progress_bar:
        groups = tqdm(groups, desc="Calculating Scalability. Completed Algorithms: ")

    for name, a_df in groups:
        result = a_df["Duration (Seconds)"].mean()
        result_str = datetime.strftime(datetime.utcfromtimestamp(result), '%H:%M:%S')
        scalabilities[name] = {"avg_seconds": result, "str": result_str}
    return scalabilities

def plot_scalability(scalabilities, dfs, out_path, lag_window, colors, order):
    # Boxplot of Mean Absolute Duration
    scalability_plot_df = pd.DataFrame([
        {
            "Algorithm": name,
            "Duration": duration / 60
        }
        for name, a_df in dfs
        for duration in a_df["Duration (Seconds)"].tolist()
    ])

    fig, ax = plt.subplots(figsize=(10, 5))
    ax.set_xlabel("Duration [Minutes]")

    sorted_names = list(scalabilities.items())
    sorted_names.sort(key=lambda x: x[1]["avg_seconds"])

    palette = sns.color_palette(colors)
    ax = sns.boxplot(
        data=scalability_plot_df,
        order=order,
        x="Duration",
        y="Algorithm",
        palette = palette,
        width=1,
        ax=ax,
        fliersize=0
    )
    #ax.figure.set_size_inches(14, 4)
    ax.spines.right.set_visible(False)
    ax.spines.top.set_visible(False)
    ax.tick_params(labelsize=16)
    ax.set_xlabel(ax.get_xlabel(), size = 18)
    ax.set_ylabel(ax.get_ylabel(), size = 18)
    plt.xlabel("Duration [Minutes]")

    if not os.path.exists(f"{out_path}/Scalability{lag_window}"):
        os.makedirs(f"{out_path}/Scalability{lag_window}")
    plt.savefig(f"{out_path}/Scalability{lag_window}/scalability.pdf", bbox_inches="tight", format="pdf")

    # As Barplot
    scalability_barplot_df = pd.DataFrame([
        {
            "Algorithm": name,
            "Duration": duration / 60
        }
        for name, a_df in dfs
        for duration in a_df["Duration (Seconds)"].tolist()
    ])

    ax = plt.subplots(figsize=(17, 4))

    ax = sns.barplot(x="Algorithm", y="Duration", data=scalability_barplot_df,palette = palette, order=order)
    ax.figure.set_size_inches(12, 4)
    ax.spines.right.set_visible(False)
    ax.spines.top.set_visible(False)
    ax.tick_params(labelsize=16)
    ax.set_xlabel(ax.get_xlabel(), size = 18)
    ax.set_ylabel(ax.get_ylabel(), size = 18)
    plt.ylabel("Duration [Minutes]")
    plt.savefig(f"{out_path}/Scalability{lag_window}/scalability_barplot.pdf", bbox_inches="tight", format="pdf")

def calculate_rel_scalabilities(df, loglengths, loglengths_events):
    df_seconds = df.copy()
    # Add Log Length Column
    df_seconds["Log Length (Cases)"] = df_seconds[["Log Source", "Log"]].apply(axis=1, func=lambda x: loglengths[Path("EvaluationLogs", x[0], x[1]+".xes.gz")])
    df_seconds["Log Length (Events)"] = df_seconds[["Log Source", "Log"]].apply(axis=1, func=lambda x: loglengths_events[Path("EvaluationLogs", x[0], x[1]+".xes.gz")])

    seconds_per_case = dict()
    seconds_per_event = dict()

    for name, group in df_seconds.groupby("Algorithm"):
        total_seconds = group["Duration (Seconds)"].sum()

        total_cases = group["Log Length (Cases)"].sum()
        total_events = group["Log Length (Events)"].sum()

        seconds_per_case[name] = total_seconds / total_cases
        seconds_per_event[name] = total_seconds / total_events

    df_seconds["Milliseconds per Event"] = df_seconds.apply(lambda x: (x["Duration (Seconds)"]*1000) / x["Log Length (Events)"], axis=1)
    return df_seconds, seconds_per_case, seconds_per_event

def plot_rel_scalability(df_seconds, out_path, lag_window, colors, order):
    fig, ax = plt.subplots(figsize=(10, 5))
    ax.set_xlabel("Milliseconds per Event")

    palette = sns.color_palette(colors)
    ax = sns.boxplot(
        data=df_seconds,
        order=order,
        x="Milliseconds per Event",
        y="Algorithm",
        palette = palette,
        width=1,
        ax=ax,
        fliersize=0

    )

    ax.set_xscale("log")

    ax.spines.right.set_visible(False)
    ax.spines.top.set_visible(False)
    ax.tick_params(labelsize=16)
    ax.set_xlabel(ax.get_xlabel(), size = 18)
    ax.set_ylabel(ax.get_ylabel(), size = 18)
    plt.xlabel("Milliseconds per Event")

    if not os.path.exists(f"{out_path}/Scalability{lag_window}"):
        os.makedirs(f"{out_path}/Scalability{lag_window}")
    plt.savefig(f"{out_path}/Scalability{lag_window}/scalability_mseconds_per_event.pdf", bbox_inches="tight", format="pdf")

    ax = plt.subplots(figsize=(17, 4))
    ax = sns.barplot(x="Algorithm", y="Milliseconds per Event", data=df_seconds, palette = palette, order=order)
    ax.figure.set_size_inches(12, 4)
    ax.spines.right.set_visible(False)
    ax.spines.top.set_visible(False)
    ax.tick_params(labelsize=16)
    ax.set_xlabel(ax.get_xlabel(), size = 18)
    ax.set_ylabel(ax.get_ylabel(), size = 18)
    ax.set_yscale("log")
    plt.ylabel("Milliseconds per Event")

    if not os.path.exists(f"{out_path}/Scalability{lag_window}"):
        os.makedirs(f"{out_path}/Scalability{lag_window}")
    plt.savefig(f"{out_path}/Scalability{lag_window}/scalability_mseconds_per_event_barplot.pdf", bbox_inches="tight", format="pdf")

def calculate_parameter_sensitivity(df, versatility_recall_dicts, computed_accuracy_dicts, scaled_latency_dicts, show_progress_bar: bool = False):
    versatility_score_per_param = {
        name: {
            params: np.nanmean(list(versatility_recall_dicts[name][params].values()))
            for params in versatility_recall_dicts[name].keys()
        }
        for name in versatility_recall_dicts.keys()
    }

    algnames = df["Algorithm"].unique()
    if show_progress_bar:
        algnames = tqdm(algnames, "Calculating Parameter Sensitivity. Completed Algorithms: ")

    sensitivities = dict()
    for name in algnames:
        _sensitivities = dict()
        acc = computed_accuracy_dicts[name]
        lat = scaled_latency_dicts[name]
        vers = versatility_score_per_param[name]
        for param_choice in acc.keys():
            sensitivity = harmonic_mean([acc[param_choice], vers[param_choice], lat[param_choice]])
            _sensitivities[param_choice] = sensitivity
        sensitivities[name] = _sensitivities

    return sensitivities

def plot_parameter_sensitivity(sensitivities, out_path, lag_window, colors, order):
    sens_df = pd.DataFrame([
        {
            "Parameters": param,
            "Algorithm": name,
            "Sensitivity Score": sens
        }
        for name, sens_dict in sensitivities.items()
        for param, sens in sens_dict.items()
    ])

    palette = sns.color_palette(colors)

    fig, ax = plt.subplots(figsize=(10, 5))
    ax = sns.boxplot(
        data=sens_df,
        x="Sensitivity Score",
        y="Algorithm",
        palette = palette,
        ax=ax,
        fliersize=0,
        order=order
    )
    ax.figure.set_size_inches(7, 4)
    ax.spines.right.set_visible(False)
    ax.spines.top.set_visible(False)
    ax.tick_params(labelsize=16)
    ax.set_xlabel("Performance", size = 18)
    ax.set_ylabel(ax.get_ylabel(), size = 18)

    if not os.path.exists(f"{out_path}/Parameter_Sensitivity{lag_window}"):
        os.makedirs(f"{out_path}/Parameter_Sensitivity{lag_window}")
    plt.savefig(f"{out_path}/Parameter_Sensitivity{lag_window}/param_sensitivity.pdf", bbox_inches="tight", format="pdf")

def calculate_parameter_sensitivity_iqr(sensitivities):
    sensitivity_iqrs = dict()
    for name, sens in sensitivities.items():
        _iqr = iqr(list(sens.values()), nan_policy="omit")
        sensitivity_iqrs[name] = _iqr
    return sensitivity_iqrs

def calc_harm_means(dataframe, min_support, lag_window, show_progress_bar: bool = False):
    robustnesses = dict()

    groups = dataframe.groupby("Noise Level")
    if show_progress_bar:
        groups = tqdm(groups, "Calculating Harmonic Means for Robustness. Noise Levels Completed:")

    for noise_level, noise_df in groups:
        # Calculate accuracy over these logs
        accuracies, _, _, _, _ = calculate_accuracy_metric_df(noise_df, lag_window, show_progress_bar=False)
        latencies, _, _, _ = calculate_latency(noise_df, lag_window, min_support=min_support, show_progress_bar=False)
        versatilities, _, _ = calc_versatility(noise_df, lag_window, show_progress_bar=False)

        assert accuracies.keys() == latencies.keys() == versatilities.keys()

        robustnesses[noise_level] = {
            approach: harmonic_mean([accuracies[approach], latencies[approach], versatilities[approach]])
            for approach in accuracies.keys()
        }
    return robustnesses

def convert_harm_mean_to_auc(means, div_zero_default=0):
    ## Conventions:
        # If any harmonic mean is nan, then set it to 0 instead

    def _convert_nan_to_zero(x):
        if np.isnan(x):
            x = 0
        return x

    # means maps {noise_level: {approach: robustness}}
    # reformat to {approach: [(noise_level, robustness)]}
    means = {
        approach: [
            (int(noise_level), _convert_nan_to_zero(means[noise_level][approach]))
            for noise_level in means.keys()
        ]
        for approach in means["0"].keys()
    }

    robustnesses = dict()
    for approach in means.keys():
        points = sorted(means[approach], key=lambda x: x[0])

        # Initial Robustness - Use to model "Ideal Robustness"
        initial_robustness = points[0][1] # Robustness for lowest (0%) noise
        ideal_auc = initial_robustness * (points[-1][0] - points[0][0]) # AUC if it was always exactly as good as for the lowest noise level. If performance increases with higher noise, the achieved AUC might be larger than ideal AUC
        prev_noise, prev_robust = points[0]
        auc = 0
        for noise, robust in points[1:]:
            delta_noise = noise-prev_noise
            sum_robust = prev_robust+robust

            # "ROC Index"
            area = (delta_noise * sum_robust) / 2# Area under curve for this segment
            auc += area

            prev_noise = noise
            prev_robust = robust

        if ideal_auc != 0:
            robustnesses[approach] = auc / ideal_auc
        else:
            robustnesses[approach] = div_zero_default
    return robustnesses

def _plot_robustness_one(robustness_df, logset, color_map):
    fig, axs = plt.subplots(2,4, figsize=(12,6))
    #fig.suptitle(f"Robustness of Approaches for {logset} Logs")

    for idx, name in enumerate(robustness_df["Approach"].unique()):
        row_offset = idx // 4
        col_offset = idx % 4
        ax = axs[row_offset, col_offset]
        relevant_df = robustness_df[(robustness_df["Approach"] == name) & (robustness_df["Log Set"] == logset)]
        sns.pointplot(x="Noise Level", y="Robustness", data=relevant_df, errorbar=None,color= color_map[name], ax=ax)#, title=name)
        ax.set_ylim(0,1)
        ax.set_title(name, size = 16)
        ax.spines.right.set_visible(False)
        ax.spines.top.set_visible(False)
        ax.tick_params(labelsize=16)
        ax.set_xlabel(ax.get_xlabel() if row_offset == 1 else "", size = 18)
        ax.set_ylabel("Performance" if col_offset == 0 else "", size = 18)
        robustness_of_zero_noise = min(zip(relevant_df["Noise Level"], relevant_df["Robustness"]), key=lambda x: x[0])[1]
        if np.isnan(robustness_of_zero_noise):
            robustness_of_zero_noise = 0
        # plt.axhline(y=robustness_of_zero_noise, color='r', linestyle='-', ax=ax)
        ax.hlines(robustness_of_zero_noise, color=color_map[name], linestyle='dashed', xmin=0, xmax=len(relevant_df["Noise Level"].unique())-1)
        fig.tight_layout()
        #ax.grid()
        fig.set_facecolor('0.9')
    return fig, axs

def plot_robustness(means_ostovar, means_ceravolo, out_path, lag_window, color_map):
    plot_df_robust = pd.DataFrame(
        ([
            {
                "Log Set": "Ostovar",
                "Approach": name,
                "Noise Level": int(noise_level),
                "Robustness": robustness
            }
            for noise_level, robust_dict  in means_ostovar.items()
            for name, robustness in robust_dict.items()
        ] + [
            {
                "Log Set": "Ceravolo",
                "Approach": name,
                "Noise Level": int(noise_level),
                "Robustness": robustness
            }
            for noise_level, robust_dict  in means_ceravolo.items()
            for name, robustness in robust_dict.items()
        ])
    )

    if not os.path.exists(f"{out_path}/Robustness{lag_window}"):
        os.makedirs(f"{out_path}/Robustness{lag_window}")

    fig_ostovar, axs_ostovar = _plot_robustness_one(plot_df_robust, "Ostovar", color_map)
    fig_ostovar.savefig(f"{out_path}/Robustness{lag_window}/robustness_ostovar_logs.pdf", bbox_inches="tight", format="pdf")

    fig_ceravolo, axs_ceravolo = _plot_robustness_one(plot_df_robust, "Ceravolo", color_map)
    fig_ceravolo.savefig(f"{out_path}/Robustness{lag_window}/robustness_ceravolo_logs.pdf", bbox_inches="tight", format="pdf")

    plt.close('all')

    fig_ostovar_in_one = _plot_robustness_in_one(plot_df_robust, "Ostovar", color_map)
    fig_ostovar_in_one.savefig(f"{out_path}/Robustness{lag_window}/robustness_ostovar_logs_all_in_one.pdf", bbox_inches="tight", format="pdf")

    fig_ceravolo_in_one = _plot_robustness_in_one(plot_df_robust, "Ceravolo", color_map)
    fig_ceravolo_in_one.savefig(f"{out_path}/Robustness{lag_window}/robustness_ceravolo_logs_all_in_one.pdf", bbox_inches="tight", format="pdf")

def _plot_robustness_in_one(robustness_df, logset, color_map):
    for _, name in enumerate(robustness_df["Approach"].unique()):
        relevant_df = robustness_df[(robustness_df["Approach"] == name) & (robustness_df["Log Set"] == logset)]
        ax = sns.pointplot(x="Noise Level", y="Robustness", data=relevant_df, errorbar=None,color= color_map[name])#, title=name)
        ax.set_ylim(0,1)
        ax.figure.set_size_inches(5, 12)
        #ax.set_title(logset+" Logs with Different Noise Levels", size = 18)
        ax.set_xlabel("Noise (%)")
        ax.spines.right.set_visible(False)
        ax.spines.top.set_visible(False)
        ax.tick_params(labelsize=16)
        ax.set_xlabel(ax.get_xlabel(), size = 18)
        ax.set_ylabel(ax.get_ylabel(), size = 18)
        robustness_of_zero_noise = min(zip(relevant_df["Noise Level"], relevant_df["Robustness"]), key=lambda x: x[0])[1]
        if np.isnan(robustness_of_zero_noise):
            robustness_of_zero_noise = 0
        # plt.axhline(y=robustness_of_zero_noise, color='r', linestyle='-', ax=ax)
        ax.hlines(robustness_of_zero_noise, color=color_map[name], linestyle='dashed', xmin=0, xmax=len(relevant_df["Noise Level"].unique())-1)
    return ax.figure


import pytest

from django.contrib.auth.models import User

@pytest.mark.skip('not now')
@pytest.mark.django_db
def test_new_user(user_factory):
    user = user_factory.create()
    count = User.objects.all().count()
    print(count)
    assert count == 1
    print(user.username)
    assert True


# -*- coding: utf-8 -*-
import ckan.plugins.toolkit as tk
import logging

log = logging.getLogger(__name__)

@tk.side_effect_free
def deleted_user_list(context, data_dict):
    '''Return a list of the site's deketed user accounts.
    '''
    model = context['model']
    tk.check_access('deleted_user_list', context, data_dict)

    query = model.Session.query(model.User.name)
    # Filter for deleted users
    query = query.filter(model.User.state == model.State.DELETED)
    users_list = []
    for user in query.all():
        users_list.append(user[0])

    return users_list