import torch
import numpy as np
import pandas as pd
from training.model import SeqLSTM
from training.data_utils import get_data_loader_chr
from training.data_utils import get_samples_sparse, get_cumpos, contactProbabilities
import time
from torch.utils.tensorboard import SummaryWriter
import torch.nn.functional as F
from training.config import Config
from analyses.feature_attribution.tf import TFChip
from analyses.classification.domains import Domains
from analyses.plot.plot_utils import get_heatmaps, simple_plot, indices_diff_mat, scatter_tal_lm
from training.test_model import test_model
from random import sample

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

class Knockout():
    """
    Class to perform knockout experiments.
    Includes methods that help you do knockout.
    """

    def __init__(self, cfg, chr):
        self.cfg = cfg
        self.chr = chr
        self.cell = cfg.cell
        self.res = cfg.resolution
        self.sizes_path = self.cfg.hic_path + self.cfg.sizes_file
        self.sizes = np.load(self.sizes_path, allow_pickle=True).item()
        self.hek_file = cfg.hic_path + cfg.cell + "/HEK239T-WT.matrix"
        self.tal1ko_file = cfg.hic_path + cfg.cell + "/Tal1KO.matrix"
        self.lmo2ko_file = cfg.hic_path + cfg.cell + "/Lmo2KO.matrix"

    def get_trained_representations(self, method="hiclstm"):
        """
        get_trained_representations(method) -> Array, int
        Gets fully trained representations for given method, cell type and chromosome.
        obtain sniper and sca representations from respective methods.
        Should contain SNIPER and SCA positions end internal representations.
        Args:
            method (string): one of hiclstm, sniper, sca
        """

        pred_data = pd.read_csv(
            self.cfg.output_directory + "%s_%s_predictions_chr%s.csv" % (method, self.cell, str(self.chr)),
            sep="\t")
        pred_data = pred_data.drop(['Unnamed: 0'], axis=1)
        representations, start, stop = self.convert_df_to_np(pred_data, method=method)
        pred_data = pred_data.filter(['i', 'j', 'v', 'pred'], axis=1)
        return representations, start, stop, pred_data

    def get_tadbs(self):
        """
        get_tadbs() -> Array
        Gets TAD Boundaries to knockout.
        Args:
            NA
        """

        dom_ob = Domains(cfg, chr, mode="ko")
        tf_ob = TFChip(cfg, chr)
        tadbs = dom_ob.get_tad_boundaries(tf_ob, ctcf="negative")
        cum_pos = get_cumpos(self.cfg, self.chr)
        tadbs = tadbs + cum_pos
        tadbs = np.array(tadbs)
        return tadbs

    def get_ctcf_indices(self):
        """
        get_ctcf_indices() -> Array
        Gets CTCF positions to knockout.
        Args:
            NA
        """

        "gets CTCF positions"
        ctcf_ob = TFChip(cfg, chr)
        data = ctcf_ob.get_ctcf_data()
        data = data.filter(['start'], axis=1)

        "converts to cumulative indices"
        cum_pos = get_cumpos(self.cfg, self.chr)
        data["start"] = data["start"] + cum_pos
        indices = np.array(data["start"])
        return indices

    def convert_df_to_np(self, pred_data, method="hiclstm"):
        """
        convert_df_to_np(pred_data, method) -> DataFrame
        Convert dataframe to np array. Easier to manipulate and provide to torch later.
        Args:
            pred_data (DataFrame): Frame containing get_trained_representations.
            method (string): one of hiclstm, sniper, sca
        """

        "assign start and stop"
        start = min(int(pred_data['i'].min()), int(pred_data['j'].min()))
        stop = max(int(pred_data['i'].max()), int(pred_data['j'].max()))

        try:
            "try loading representations"
            embed_rows = np.load(
                self.cfg.output_directory + "%s_rep_%s_chr%s.npy" % (method, self.cell, str(self.chr)))
        except:
            "initialize"
            nrows = int(stop - start) + 1
            embed_rows = np.zeros((nrows, self.cfg.pos_embed_size))

            i_old = 0
            j_old = 0
            for r in range(len(pred_data)):
                i_new = int(pred_data.loc[r, "i"])

                "skip already seen positions"
                if i_new == i_old:
                    continue
                else:
                    "assign representations"
                    i_old = i_new
                    if np.all((embed_rows[i_new - start, :] == 0)):
                        col = list(np.arange(self.cfg.pos_embed_size))
                        col = [str(x) for x in col]
                        embed_rows[i_new - start, :] = np.array(pred_data.loc[r, col])

                "repeat for j"
                j_new = int(pred_data.loc[r, "j"])
                if j_new == j_old:
                    continue
                else:
                    "assign representations"
                    j_old = j_new
                    if np.all((embed_rows[j_new - start, :] == 0)):
                        col = list(np.arange(self.cfg.pos_embed_size, 2 * self.cfg.pos_embed_size))
                        col = [str(x) for x in col]
                        embed_rows[j_new - start, :] = np.array(pred_data.loc[r, col])

            np.save(self.cfg.output_directory + "%s_rep_%s_chr%s.npy" % (method, self.cell, str(self.chr)), embed_rows)
        return embed_rows, start, stop

    def normalize_embed(self, representations, zero_embed):
        """
        normalize_embed(representations, zero_embed) -> Array, Array
        Normalize each row separately.
        Args:
            representations (Array): Array of representations to normalize.
            zero_embed (Array): pading representation
        """

        "normalize representations"
        for n in range(len(representations)):
            norm = np.linalg.norm(representations[n, :])
            if norm == 0:
                continue
            else:
                representations[n, :] = representations[n, :] / norm

        "normalize padding"
        norm = np.linalg.norm(zero_embed)
        zero_embed = zero_embed / norm
        return representations, zero_embed

    def ko_representations(self, representations, start, indices, zero_embed, mode="average"):
        """
        ko_representations(representations, start, indices, zero_embed, mode) -> Array, Array
        Alter representations to feed to knockout.
        Args:
            representations (Array): Array of representations.
            start (ind): start indice in chromosome.
            indices (list): indices to knockout
            zero_embed (Array): Padding representation
            mode (string): one of average, zero, padding, shift, normalize, or reverse
        """

        window = self.cfg.ko_window
        size = len(representations)

        if isinstance(indices, (int, np.integer)):
            indices = [indices]

        "alter according to mode in config"
        for ind in indices:
            if mode == "average":
                if ind - start - window < 0 or ind - start + window > size:
                    window = int(window // 2)

                window_left_arr = representations[ind - start - window: ind - start, :].copy()
                window_right_arr = representations[ind - start + 1: ind - start + window + 1, :].copy()

                window_arr_avg = np.stack((window_left_arr, window_right_arr)).mean(axis=0).mean(axis=0)
                representations[ind - start, :] = window_arr_avg
            elif mode == "zero":
                representations[ind - start, :] = np.zeros((1, cfg.pos_embed_size))
            elif mode == "shift":
                representations[ind - start:size - 1, :] = representations[ind - start + 1:size, :]
                representations[size - 1, :] = np.zeros((1, cfg.pos_embed_size))
            elif mode == "padding":
                representations[ind - start, :] = zero_embed[:cfg.pos_embed_size]

        if mode == "reverse":
            representations = np.fliplr(representations)
            zero_embed = np.flip(zero_embed)
        elif mode == "normalize":
            representations, zero_embed = self.normalize_embed(representations, zero_embed)

        return representations, zero_embed

    def compute_kodiff(self, pred_data, ko_pred_df, ind):
        """
        compute_kodiff(pred_data, ko_pred_df, indices) -> Array
        Compute difference between predicted contacts after and before knockout
        Args:
            pred_data (DataFrame): Frame containing predictions before knockout
            ko_pred_df (DataFrame): Frame containing predictions after knockout.
            indices (list): indices that were knocked out
        """

        "initialize"
        ko_diffs = np.zeros((11,))
        win = self.cfg.ko_increment
        diff = np.arange(0, 101, 10)

        "compute diff"
        for j, d in enumerate(diff):
            "take subset of knockout data in window"
            if j == 0:
                subset_og = pred_data.loc[pred_data["i"] == ind]
            else:
                subset_og = pred_data.loc[
                    ((pred_data["i"] <= ind + j * win) & (pred_data["i"] > ind + (j - 1) * win))
                    | ((pred_data["i"] >= ind - j * win) & (pred_data["i"] < ind - (j - 1) * win))]
            if subset_og.empty:
                continue

            "take subset of original data in window"
            if j == 0:
                subset_ko = ko_pred_df.loc[ko_pred_df["i"] == ind]
            else:
                subset_ko = ko_pred_df.loc[
                    ((ko_pred_df["i"] <= ind + j * win) & (ko_pred_df["i"] > ind + (j - 1) * win))
                    | ((ko_pred_df["i"] >= ind - j * win) & (ko_pred_df["i"] < ind - (j - 1) * win))]

            "compute mean diff in window"
            merged_df = pd.merge(subset_og, subset_ko, on=["i", "j"])
            merged_df = merged_df.filter(['i', 'j', 'pred', 'ko_pred'], axis=1)
            mean_diff = np.mean(np.array(merged_df["ko_pred"]) - np.array(merged_df["pred"]))
            ko_diffs[j] = mean_diff
        return ko_diffs

    def perform_ko(self, model):
        """
        perform_ko(model) -> Array
        Loads data for chromosome. Loads representations. Alters representations.
        Gets padding representation. Runs through decoder. Computes mean diff between WT and KO.
        Saves predictions.
        Args:
            model (SeqLSTM): model to use for knockout.
        """

        cfg = self.cfg

        "load data"
        if cfg.run_tal and cfg.hnisz_region == "tal1":
            self.cfg.get_tal1_only = True
            data_loader = self.prepare_tal1_lmo2()
        elif cfg.run_tal and cfg.hnisz_region == "lmo2":
            self.cfg.get_lmo2_only = True
            data_loader = self.prepare_tal1_lmo2()
        else:
            data_loader = get_data_loader_chr(cfg, self.chr, shuffle=False)

        "get zero embed"
        cfg.full_test = False
        cfg.compute_pca = False
        cfg.get_zero_pred = True
        zero_embed = test_model(model, cfg, self.chr)

        "get knockout indices depending on experiment"
        if cfg.run_tal:
            if cfg.hnisz_region == "tal1":
                cfg.ko_experiment = "ctcf"
                indices = cfg.tal1ko_indices
            elif cfg.hnisz_region == "lmo2":
                cfg.ko_experiment = "ctcf"
                indices = np.array(cfg.lmo2ko_indices) + get_cumpos(cfg, 11)
        else:
            if cfg.ko_experiment == "ctcf":
                if cfg.ctcf_indices == "all":
                    indices = ko_ob.get_ctcf_indices()
                    indices = sample(list(indices), 10)
                else:
                    indices = ko_ob.cfg.ctcf_indices_22
            elif cfg.ko_experiment == "foxg1":
                indices = cfg.foxg1_indices
            elif cfg.ko_experiment == "tadbs":
                indices = ko_ob.get_tadbs()

        "plotting and metrics"
        n_indices = len(indices)
        diff_list = np.zeros((n_indices, 11))
        diff_mat = np.zeros((n_indices, 200, 200))
        "run for all indices"
        for i, indice in enumerate(indices):
            "get representations"
            representations, start, stop, pred_data = self.get_trained_representations(method="hiclstm")

            "alter representations"
            representations, zero_embed = self.ko_representations(representations, start, indice, zero_embed,
                                                                  mode=cfg.ko_mode)

            if self.cfg.load_ko:
                ko_pred_df = pd.read_csv(cfg.output_directory + "hiclstm_%s_afko_chr%s.csv" % (cfg.cell, str(chr)),
                                         sep="\t")
            else:
                "run through model using altered representations, save ko predictions"
                _, ko_pred_df = model.perform_ko(data_loader, representations, start, zero_embed, mode="ko")
                if self.cfg.save_kopred:
                    ko_pred_df.to_csv(cfg.output_directory + "hiclstm_%s_afko_chr%s.csv" % (cfg.cell, str(chr)),
                                      sep="\t")

            "compute difference between WT and KO predictions"
            if self.cfg.compute_avg_diff:
                ko_diffs = self.compute_kodiff(pred_data, ko_pred_df, indice)
                diff_list[i] = ko_diffs

            "get merged heatmap"
            pred_data = pd.merge(pred_data, ko_pred_df, on=["i", "j"])
            pred_data = pred_data.rename(columns={"ko_pred": "v"})
            hic_mat, st = get_heatmaps(pred_data, no_pred=False)
            # simple_plot(hic_mat, mode="reds")

            "get diff mat"
            hic_win = indices_diff_mat(indice, st, hic_mat, mode=cfg.ko_experiment)
            n_win = len(hic_win)
            diff_mat[i, :n_win, :n_win] = hic_win

        diff_mat = diff_mat.mean(axis=0)
        ko = np.triu(diff_mat)
        pred = np.tril(diff_mat).T
        diff_mat = ko - pred
        simple_plot(diff_mat, mode="diff")
        np.save(cfg.output_directory + "tad_diff_zero_ctctn.npy", diff_mat)
        mean_diff = np.mean(diff_list, axis=1)
        return mean_diff, ko_pred_df, pred_data

    def change_index(self, list_split):
        """
        change_index(list_split) -> list, list
        get locations from index.
        Args:
            list_split (list): list
        """

        "format index"
        temp = [k.split('|')[-1] for k in list_split]
        chr_list = []
        index_list = []
        for t in temp:
            index = t.split(':')
            chr_list.append(index[0])
            index_list.append(index[1].split('-'))

        "prepare locations list"
        loc_list = []
        for ind in index_list:
            loc = int(((int(ind[0]) + int(ind[1])) / 2) // 10000)
            loc_list.append(loc)

        return loc_list, chr_list

    def convert_to_hic_format(self):
        """
        convert_to_hic_format() -> No return object.
        Assigns positions and chr. Convert 5C to Hi-C like format.
        Args:
            NA.
        """

        if self.cfg.tal_mode == "wt":
            hek_mat = pd.read_csv(self.hek_file, sep="\t")
        elif self.cfg.tal_mode == "tal1_ko":
            hek_mat = pd.read_csv(self.tal1ko_file, sep="\t")
        elif self.cfg.tal_mode == "lmo2_ko":
            hek_mat = pd.read_csv(self.lmo2ko_file, sep="\t")

        "get positions"
        index, chr_list = self.change_index(list(hek_mat.index))
        columns, _ = self.change_index(hek_mat.columns)

        "assign rows, columns and chr"
        hek_mat.index = index
        hek_mat.columns = columns
        hek_mat["chr"] = chr_list

        "get matrices for TAL1 and LMO2"
        tal1_mat = hek_mat.loc[hek_mat["chr"] == "chr1"]
        tal1_mat = tal1_mat.iloc[:, 0:285]
        lmo2_mat = hek_mat.loc[hek_mat["chr"] == "chr11"]
        lmo2_mat = lmo2_mat.iloc[:, 286:632]
        tal1_mat = tal1_mat.groupby(level=0, axis=1).sum()
        tal1_mat = tal1_mat.groupby(level=0, axis=0).sum()
        lmo2_mat = lmo2_mat.groupby(level=0, axis=1).sum()
        lmo2_mat = lmo2_mat.groupby(level=0, axis=0).sum()

        "prepare data in the form of Hi-C"
        tal_i = list(tal1_mat.index)
        tal_j = tal1_mat.columns
        lmo2_i = list(lmo2_mat.index)
        lmo2_j = lmo2_mat.columns

        tal_df = pd.DataFrame(columns=["i", "j", "v"])
        for i in tal_i:
            for j in tal_j:
                tal_df = tal_df.append({"i": i, "j": j, "v": tal1_mat.loc[i][j]}, ignore_index=True)

        lmo2_df = pd.DataFrame(columns=["i", "j", "v"])
        for i in lmo2_i:
            for j in lmo2_j:
                lmo2_df = lmo2_df.append({"i": i, "j": j, "v": lmo2_mat.loc[i][j]}, ignore_index=True)

        "save data"
        if self.cfg.tal_mode == "wt":
            tal_df.to_csv(cfg.hic_path + cfg.cell + "/tal_df.txt", sep="\t")
            lmo2_df.to_csv(cfg.hic_path + cfg.cell + "/lmo2_df.txt", sep="\t")
        else:
            tal_df.to_csv(cfg.output_directory + "tal1_ko.txt", sep="\t")
            lmo2_df.to_csv(cfg.output_directory + "lmo2_ko.txt", sep="\t")

    def prepare_tal1_lmo2(self):
        """
        prepare_tal1_lmo2(cfg) -> DataLoader
        prepare dataloader to train.
        Args:
            cfg (Config): config to be used to run the experiment.
        """

        "load Hi-C like data"
        tal_df = pd.read_csv(cfg.hic_path + cfg.cell + "/tal_df.txt", sep="\t")
        lmo2_df = pd.read_csv(cfg.hic_path + cfg.cell + "/lmo2_df.txt", sep="\t")

        "preprocess"
        tal_df = tal_df.drop(['Unnamed: 0'], axis=1)
        lmo2_df = lmo2_df.drop(['Unnamed: 0'], axis=1)
        tal_df[['i', 'j']] = tal_df[['i', 'j']].astype('int64')
        lmo2_df[['i', 'j']] = lmo2_df[['i', 'j']].astype('int64')

        "prepare indices and values for TAL1 in chromosome 1"
        values = torch.empty(0, cfg.sequence_length)
        input_idx = torch.empty(0, cfg.sequence_length, 2)
        input_idx_tal1, values_tal1 = get_samples_sparse(tal_df, 1, cfg)
        values_tal1 = F.pad(input=values_tal1, pad=(0, 4, 0, 0), mode='constant', value=0)
        input_idx_tal1 = F.pad(input=input_idx_tal1, pad=(0, 0, 0, 4, 0, 0), mode='constant', value=0)
        values = torch.cat((values, values_tal1.float()), 0)
        input_idx = torch.cat((input_idx, input_idx_tal1), 0)

        if self.cfg.get_tal1_only:
            "create tal dataloader"
            dataset = torch.utils.data.TensorDataset(input_idx, values)
            data_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=cfg.batch_size, shuffle=True)
            return data_loader

        if self.cfg.get_lmo2_only:
            values = torch.empty(0, cfg.sequence_length)
            input_idx = torch.empty(0, cfg.sequence_length, 2)

        "prepare indices and values for LMO2 in chromosome 11"
        input_idx_lmo2, values_lmo2 = get_samples_sparse(lmo2_df, 11, cfg)
        values = torch.cat((values, values_lmo2.float()), 0)
        input_idx = torch.cat((input_idx, input_idx_lmo2), 0)

        "create dataloader"
        dataset = torch.utils.data.TensorDataset(input_idx, values)
        data_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=cfg.batch_size, shuffle=True)

        return data_loader

    def train_tal1_lmo2(self, model):
        """
        train_tal1_lmo2(model, cfg) -> No return object
        Train model on 5C data from TAL1 and LMO2 regions.
        Args:
            model (SeqLSTM): Model to be used to train on 5C data
        """

        "summary writer"
        timestr = time.strftime("%Y%m%d-%H%M%S")
        writer = SummaryWriter('./tensorboard_logs/' + cfg.model_name + timestr)

        "initialize optimizer and prepare dataloader"
        self.cfg.get_tal1_only = False
        self.cfg.get_lmo2_only = False
        optimizer, criterion = model.compile_optimizer()
        data_loader = self.prepare_tal1_lmo2()

        "train and save the model"
        model.train_model(data_loader, criterion, optimizer, writer)
        torch.save(model.state_dict(), cfg.model_dir + cfg.model_name + '.pth')

    def test_tal1_lmo2(self, model):
        """
        test_tal1_lmo2(model) -> DataFrame
        Test model on 5C data from TAL1 and LMO2 regions.
        Args:
            model (SeqLSTM):  Model to be used to test on 5C data
        """

        "prepare dataloader"
        data_loader = self.prepare_tal1_lmo2()

        "test model"
        self.cfg.full_test = True
        self.cfg.compute_pca = False
        self.cfg.get_zero_pred = False
        _, _, _, pred_df, _ = model.test(data_loader)

        "save predictions"
        pred_df.to_csv(self.cfg.output_directory + "hiclstm_%s_predictions_chr%s.csv" % (self.cell, str(self.chr)),
                       sep="\t")
        return pred_df

    def perform_tal1_ko(self, model):
        """
        perform_tal1_ko(model) -> DataFrame
        Performs knockout of selected sites in TAL1 and LMO2 regions.
        Args:
            model (SeqLSTM):  Model to be used to test on 5C data
        """

        "save representations"
        self.chr = 1
        self.cfg.get_tal1_only = True
        ko_ob.test_tal1_lmo2(model)

        "perform ko"
        self.cfg.hnisz_region = "tal1"
        _, ko_pred_df, _ = self.perform_ko(model)
        return ko_pred_df

    def perform_lmo2_ko(self, model):
        """
        perform_tal1_ko(model) -> DataFrame
        Performs knockout of selected sites in TAL1 and LMO2 regions.
        Args:
            model (SeqLSTM):  Model to be used to test on 5C data
        """

        "save representations"
        self.chr = 11
        self.cfg.get_lmo2_only = True
        ko_ob.test_tal1_lmo2(model)

        "perform ko"
        self.cfg.hnisz_region = "lmo2"
        _, ko_pred_df, _ = self.perform_ko(model)

        return ko_pred_df

    def run_tal_experiment(self):
        pred_df = None
        ko_pred_df = None
        tal1_data = None
        lmo2_data = None

        "load model"
        model = SeqLSTM(self.cfg, device).to(device)
        model.load_weights()

        if cfg.tal_pre:
            "prepare tal1 and lmo2 data"
            self.convert_to_hic_format()

        if cfg.tal_train:
            "train 5C data"
            ko_ob.train_tal1_lmo2(model)

        if cfg.tal_test:
            "test tal1 and lmo2 regions"
            ko_ob.cfg.get_tal1_only = False
            ko_ob.cfg.get_lmo2_only = False
            pred_df = ko_ob.test_tal1_lmo2(model)

        if cfg.perform_tal1_ko:
            ko_pred_df = ko_ob.perform_tal1_ko(model)

        if cfg.perform_lmo2_ko:
            ko_pred_df = ko_ob.perform_lmo2_ko(model)

        if cfg.compare_tal:
            "compare predictions and observed 5C"
            if pred_df is None:
                pred_df = pd.read_csv(cfg.output_directory + "%s_predictions.csv" % (cfg.cell), sep="\t")

            tal1_data = pred_df.loc[pred_df["i"] < 7000]
            lmo2_data = pred_df.loc[pred_df["i"] > 7000]
            tal1_mat, _ = get_heatmaps(tal1_data.copy(), no_pred=False)
            lmo2_mat, _ = get_heatmaps(lmo2_data.copy(), no_pred=False)

            if cfg.tal_plot_wt:
                simple_plot(tal1_mat, mode="reds")
                simple_plot(lmo2_mat, mode="reds")

        if cfg.check_ko:
            "compare ko and observed 5C"
            if cfg.tal_pre_ko:
                self.cfg.tal_mode = "tal1_ko"
                self.convert_to_hic_format()

                tal1_ko = pd.read_csv(cfg.output_directory + "tal1_ko.txt", sep="\t")
                lmo2_ko = pd.read_csv(cfg.output_directory + "lmo2_ko.txt", sep="\t")

                tal1_data["pred"] = contactProbabilities(tal1_ko["v"])
                lmo2_data["pred"] = contactProbabilities(lmo2_ko["v"])
                tal1_mat, _ = get_heatmaps(tal1_data, no_pred=False)
                lmo2_mat, _ = get_heatmaps(lmo2_data, no_pred=False)

                if cfg.tal_plot_ko:
                    simple_plot(tal1_mat, mode="reds")
                    simple_plot(lmo2_mat, mode="reds")

            if cfg.hnisz_region == "tal1":
                wt = np.load(cfg.output_directory + "tal1_wt.npy")
                ko = np.load(cfg.output_directory + "tal1_ko.npy")
            else:
                wt = np.load(cfg.output_directory + "lmo2_wt.npy")
                ko = np.load(cfg.output_directory + "lmo2_ko.npy")

            scatter_tal_lm(ko, wt)

if __name__ == '__main__':
    cfg = Config()

    "load model"
    model = SeqLSTM(cfg, device).to(device)
    model.load_weights()

    if cfg.ko_experiment == "foxg1":
        cfg.chr_test_list = cfg.foxg1_chr

    ko_pred_df = None
    for chr in cfg.chr_test_list:
        print('Knockout Start Chromosome: {}'.format(chr))
        ko_ob = Knockout(cfg, chr)

        if cfg.ko_compute_test:
            "run test if predictions not computed yet"
            test_model(model, cfg, chr)

        if cfg.perform_ko:
            "perform ko"
            ko_ob.cfg.run_tal = False
            mean_diff, ko_pred_df, pred_data = ko_ob.perform_ko(model)

        if cfg.compare_ko:
            "plot comparison"
            if ko_pred_df is None:
                _, _, _, pred_data = ko_ob.get_trained_representations(method="hiclstm")
                ko_pred_df = pd.read_csv(cfg.output_directory + "hiclstm_%s_afko_chr%s.csv" % (cfg.cell, str(chr)),
                                         sep="\t")
            pred_data = pd.merge(pred_data, ko_pred_df, on=["i", "j"])
            pred_data = pred_data.rename(columns={"ko_pred": "v"})

            hic_mat, st = get_heatmaps(pred_data, no_pred=False)
            simple_plot(hic_mat, mode="reds")
            print("done")

    "TAL1 and LMO2"
    if cfg.run_tal:
        cfg.cell = "HEK239T"
        cfg.model_name = "shuffle_" + cfg.cell
        ko_ob = Knockout(cfg, cfg.lmo2_chr)
        ko_ob.run_tal_experiment()


import sys
import os
import re
import openai
import json
import requests
import time
import pandas as pd

start_time = time.time()
df = pd.read_csv('proteins.csv', header=None, names=['search_words'])

print(df.head())

with open("prompt_chat_gpt.output.txt","w") as f:

    openai.api_key = "<replace>"
    model_engine = "gpt-3.5-turbo"
    # This specifies which GPT model to use, as there are several models available, each with different capabilities and performance characteristics.

    # open the input file and read its contents
    depth = 30
    w_list =[] # work list of proteins not prompted yet
    s_list =[] # seen list of proteins we have already prompted

    argument = sys.argv[1]

    if (not argument):
        with open('prompts.txt', 'r') as ff:
            strings = ff.readlines()
            print("no arguments")
            for string in strings:
                w_list.append(string.strip())
    else:
        w_list.append(argument.strip())

    w_list = list(set(w_list))
    print(w_list,file=f)
    leng = len(w_list)
    print("leng>",leng)
    while depth > 0 and leng > 0:
        depth = depth - 1
        target = w_list[0]
        s_list.append(target)
        del w_list[0]
        print("------------------------------------------",file=f)
        print("Target -->", target,file=f)
        model_start_time = time.time()
        response = openai.ChatCompletion.create(
            model='gpt-3.5-turbo',
            messages=[
                {"role": "system", "content": "You are a helpful assistant."},
                {"role": "user", "content": "We are interested in protein interactions.  Based on the documents you have been trained with, can you provide any information on which proteins might interact with " + target},
            ])
        model_end_time = time.time()
        elapsed_model_time = model_end_time - model_start_time
        print('Total model execution time:', elapsed_model_time, 'seconds')
        result = ''
        print(f'The response of openai model for target {target} is {response.choices}')
        for choice in response.choices:
            result += choice.message.content

            print(result, file=f)

            # Define the string to be searched
            search_string = result

            # Define an empty list to store the matches
            matches = []

            # Loop through the words in the DataFrame
            for word in df['search_words']:
                # Check if the word is in the search string
                pattern = r"\b{}\b".format(word)
                match = re.search(pattern, search_string)
                if match:
                    # If it is, append the word to the matches list
                    matches.append(word)

                # Convert the matches list into a new DataFrame
            matches_u = list(set(matches))
            matches_df = pd.DataFrame({'match_words': matches_u})

            # Print the new DataFrame
            matches_df = matches_df.drop_duplicates()
            for word in matches_df['match_words']:
                if target != word:
                    print(target,"-->", word)
                    w_list.append(word)
            w_list = list(set(w_list)) # remove duplicates before removing previously seen
            for item in s_list:
                if item in w_list:
                    w_list.remove(item)
            print(w_list, file=f)
            print("-----------------------------------------------",file=f)
            print(" ",file=f)
# get the end time
end_time = time.time()

# get the execution time
elapsed_time = end_time - start_time
print('Total block execution time:', elapsed_time, 'seconds')


from django.urls import path, re_path

from . import views
from .views import plain_text as views_plain_text

urlpatterns = [
    # Pages available to everyone.
    path('', views.index, name='index'),
    path('about', views.about, name='about'),
    # Pages / endpoints available to those who are logged in.
    # - Plain text visions, volumes, full year.
    path('vision/<int:pk>/text', views_plain_text.show_vision, name="plain_text_vision"),
    re_path('volume/(?P<volume_number>[1-9][0-9]*[a-d])/text', views_plain_text.show_single_volume,
            name="plain_text_single_volume"),
    path('volume/<int:volume_major_number>/text', views_plain_text.show_full_year,
         name="plain_text_full_year")
    # - Previews of visions, volumes, full year?
    # - DOCX (or PDF?) visions, volumes, full year.
    # Pages available to those who are logged in and have edit permissions.
    # - Add the next event
    # - Edit existing vision
    # - Add new vision
]


import os
from pyspark.sql import SparkSession
from pyspark.sql.functions import udf, from_unixtime
from pyspark.sql.types import StructType, StructField, StringType, IntegerType, TimestampType, LongType, DoubleType, FloatType

def create_spark_session():
    print("ETL.PY INFO::create_spark_session")
    spark = SparkSession \
        .builder \
        .config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:2.7.0") \
        .getOrCreate()
    return spark

def process_song_data(spark, input_data, output_data):
    print("ETL.PY INFO::process_song_data")
    # get filepath to song data file
    song_data = "s3n://udacity-dend/song_data/*/*/*"

    # create schema for song_data
    schema = StructType([
        StructField('artist_id', StringType(), True),
        StructField('artist_latitude', DoubleType(), True),
        StructField('artist_location', StringType(), True),
        StructField('artist_longitude', DoubleType(), True),
        StructField('artist_name', StringType(), True),
        StructField('duration', DoubleType(), True),
        StructField('num_songs', LongType(), True),
        StructField('song_id', StringType(), True),
        StructField('title', StringType(), True),
        StructField('year', LongType(), True)
    ])

    # read song data file
    print("ETL.PY INFO::reading in song_data from S3")
    songs_data=spark.read.json(song_data, schema)

    # create spark table view for songs data
    songs_data.createOrReplaceTempView("songs_data_table")

    print("ETL.PY INFO::creating songs_table")
    songs_table = spark.sql('''SELECT song_id, title, artist_id, year, duration
                               FROM songs_data_table''')

    # write songs table to parquet files
    print("ETL.PY INFO::writing songs_table.parquet to S3")
    songs_table.write.mode('overwrite').partitionBy("year", "artist_id").parquet(output_data + 'songs_table.parquet')

    # write a local copy to be used for creating songplays table
    print("ETL.PY INFO::writing songs_table.parquet locally")
    songs_table.write.parquet('songs_table.parquet')

    # extract columns to create artists table (artist_id, name, location, lattitude, longitude)
    print("ETL.PY INFO::creating artists_table")
    artists_table = spark.sql('''SELECT artist_id, artist_name AS name, artist_location AS location,
                                        artist_latitude AS latitude, artist_longitude AS longitude
                                 FROM songs_data_table''')

    # write artists table to parquet files
    print("ETL.PY INFO::writing artists_table.parquet to S3")
    artists_table.write.mode('overwrite').parquet(output_data + 'artists_table.parquet')

    # write a local copy to be used for creating songplays table
    print("ETL.PY INFO::writing artists_table.parquet locally")
    artists_table.write.parquet('artists_table.parquet')

def process_log_data(spark, input_data, output_data):
    print("ETL.PY INFO::process log_data")

    # get filepath to log data file
    log_data = 's3a://udacity-dend/log_data/*/*/*.json'

    # define schema for log data (spark had difficulty defining a consistent schema on all logs)
    schema = StructType([
        StructField('artist',StringType(),True),
        StructField('auth',StringType(),True),
        StructField('firstName',StringType(),True),
        StructField('gender',StringType(),True),
        StructField('itemInSession',LongType(),True),
        StructField('lastName',StringType(),True),
        StructField('length',DoubleType(),True),
        StructField('level',StringType(),True),
        StructField('location',StringType(),True),
        StructField('method',StringType(),True),
        StructField('page',StringType(),True),
        StructField('registration',DoubleType(),True),
        StructField('sessionId',LongType(),True),
        StructField('song',StringType(),True),
        StructField('status',LongType(),True),
        StructField('ts',LongType(),True),
        StructField('userAgent',StringType(),True),
        StructField('userId',StringType(),True)
    ])

    # read log data file
    print("ETL.PY INFO::reading in log_data from S3")
    log_data = spark.read.json(log_data, schema)

    # create spark table view for the log data
    log_data.createOrReplaceTempView("log_table")

    # filter by actions for song plays
    print("ETL.PY INFO::filtering out song plays where page=NextSong")
    log_data = spark.sql('''SELECT *
             FROM log_table
             WHERE page = "NextSong"''')

    # update spark table view with new filtered data
    log_data.createOrReplaceTempView("log_table")

    # extract columns for users table (user_id, first_name, last_name, gender, level)
    print("ETL.PY INFO::creating users_table")
    users_table = spark.sql('''SELECT DISTINCT userId, firstName, lastName, gender, level
                               FROM log_table''')

    # write users table to parquet files
    print("ETL.PY INFO::writing users_table.parquet to S3")
    users_table.write.mode('overwrite').parquet(output_data + 'users_table.parquet')

    # update 'ts' column from unix timestamp that was converted to seconds
    log_data = log_data.withColumn("ts", from_unixtime(log_data.ts / 1000))

    # update spark table view
    log_data.createOrReplaceTempView("log_table")

    # extract columns to create time table (ts, hour, day, week, month, year, weekday)
    print("ETL.PY INFO::creating time_table")
    time_table = spark.sql('''SELECT DISTINCT ts,
                                              date_format(ts, 'HH') AS hour,
                                              date_format(ts, 'dd') AS day,
                                              date_format(ts, 'w') AS week,
                                              date_format(ts, 'MM') AS month,
                                              date_format(ts, 'yyyy') AS year,
                                              date_format(ts, 'u') AS weekday
                              FROM log_table''')

    # write time table to parquet files partitioned by year and month
    print("ETL.PY INFO::writing time_table.parquet to S3")
    time_table.write.mode('overwrite').partitionBy("year", "month").parquet(output_data + 'time_table.parquet')

    # read in song data to use for songplays table
    print("ETL.PY INFO::reading local songs_table.parquet")
    song_df = spark.read.parquet("songs_table.parquet")
    song_df.createOrReplaceTempView("songs_table")

    # read in artist data to use for songplays table
    print("ETL.PY INFO::reading local artists_table.parquet")
    artist_df = spark.read.parquet("artists_table.parquet")
    artist_df.createOrReplaceTempView("artists_table")

    # extract columns from joined song and log datasets to create songplays table
    # (songplay_id, start_time, user_id, level, song_id, artist_id, session_id, location, user_agent)
    print("ETL.PY INFO::creating songplays_table")
    songplays_table = spark.sql('''SELECT CONCAT(e.ts, '-', e.sessionId) AS songplay_id,
                                        e.ts AS start_time,
                                        date_format(e.ts, 'MM') AS month,
                                        date_format(e.ts, 'yyyy') AS year,
                                        e.userId AS user_id,
                                        e.level AS level,
                                        sa.song_id AS song_id,
                                        sa.artist_id AS artist_id,
                                        e.sessionId AS session_id,
                                        e.location AS location,
                                        e.userAgent AS user_agent
                                   FROM log_table AS e
                                   JOIN (
                                       SELECT s.song_id, a.artist_id, s.title, a.name, s.duration
                                       FROM songs_table AS s
                                       INNER JOIN artists_table AS a
                                       ON s.artist_id = a.artist_id) AS sa
                                   ON sa.title = e.song
                                   AND sa.name = e.artist
                                   AND sa.duration = e.length
                                ''')

    # write songplays table to parquet files partitioned by year and month
    print("ETL.PY INFO::writing songplays_table.parquet to S3")
    songplays_table.write.mode('overwrite').partitionBy("year", "month").parquet(output_data + 'songplays_table.parquet')

def main():
    spark = create_spark_session()

    # set efficient file output algorithm
    spark.sparkContext._jsc.hadoopConfiguration().set("mapreduce.fileoutputcommitter.algorithm.version", "2")

    input_data = "s3a://udacity-dend/"
    output_data = "s3a://de-nano-project-4/output/"

    process_song_data(spark, input_data, output_data)
    process_log_data(spark, input_data, output_data)

if __name__ == "__main__":
    main()


import torch
from torch.utils.data import DataLoader

from semantic_segmentation.arguments import get_args
from semantic_segmentation.datasets.dataset import get_dataset
from semantic_segmentation.experiments.trainer import evaluate

args = get_args(desc="args for evaluation", mode='eval')

if args.gpu != 'cpu':
	torch.cuda.set_device(args.gpu)

_, valid_dataset, test_dataset, num_classes = get_dataset(args.dataset, None, True)

valloader = DataLoader(valid_dataset, batch_size=1, shuffle=False)
# testloader = DataLoader(test_dataset, batch_size=1, shuffle=False)

params = {
    "model": None,
    "seed": args.seed,
    'num_classes': num_classes
}

evaluate(valloader, args, params, mode='pretrain') # without teacher training

evaluate(valloader, args, params, mode='classifier') # stagewise training

evaluate(valloader, args, params, mode='simultaneous') # simultanous training

evaluate(valloader, args, params, mode='traditional-kd') # traditional-kd training


from discord.ext import commands
from util import formats
from collections import Counter

import discord
import copy

class Info(object):
    def __init__(self, bot):
        self.bot = bot

    @commands.command(pass_context=True, no_pm=True, aliases=["uinfo", "whoami"])
    async def userinfo(self, ctx, *, member : discord.Member = None):
        """Shows info about a member."""
        channel = ctx.message.channel
        if member is None:
            member = ctx.message.author

        roles = [role.name.replace('@', '@\u200b') for role in member.roles]
        shared = sum(1 for m in self.bot.get_all_members() if m.id == member.id)
        voice = member.voice_channel
        if voice is not None:
            other_people = len(voice.voice_members) - 1
            voice_fmt = '{} with {} others' if other_people else '{} by themselves'
            voice = voice_fmt.format(voice.name, other_people)
        else:
            voice = 'Not connected.'

        internal_tags = ", ".join(await self.bot.db.redis.smembers("user:{0}:tags".format(member.id)))

        entries = [
            ('Name', member.name),
            ('Tag', member.discriminator),
            ('ID', member.id),
            ('Internal Tags', internal_tags),
            ('Joined', member.joined_at),
            ('Created', member.created_at),
            ('Roles', ', '.join(roles)),
            ('Servers', '{} shared'.format(shared)),
            ('Voice', voice),
        ]

        await formats.embed(self.bot, entries, colour=member.top_role.colour, author=member, thumbnail=member.avatar_url)

    @commands.command(pass_context=True, no_pm=True, aliases=["sinfo"])
    async def serverinfo(self, ctx):
        """Shows info about the server."""
        server = ctx.message.server
        roles = [role.name.replace('@', '@\u200b') for role in server.roles]

        secret_member = copy.copy(server.me)
        secret_member.id = '0'
        secret_member.roles = [server.default_role]

        # figure out what channels are 'secret'
        secret_channels = 0
        secret_voice = 0
        text_channels = 0
        for channel in server.channels:
            perms = channel.permissions_for(secret_member)
            is_text = channel.type == discord.ChannelType.text
            text_channels += is_text
            if is_text and not perms.read_messages:
                secret_channels += 1
            elif not is_text and (not perms.connect or not perms.speak):
                secret_voice += 1

        regular_channels = len(server.channels) - secret_channels
        voice_channels = len(server.channels) - text_channels
        member_by_status = Counter(str(m.status) for m in server.members)
        member_fmt = '{0} ({1[online]} online, {1[idle]} idle, {1[offline]} offline)'
        channels_fmt = '{} Text ({} secret) / {} Voice ({} locked)'
        channels = channels_fmt.format(text_channels, secret_channels, voice_channels, secret_voice)

        entries = [
            ('Name', server.name),
            ('ID', server.id),
            ('Channels', channels),
            ('Created', server.created_at),
            ('Members', member_fmt.format(len(server.members), member_by_status)),
            ('Owner', server.owner),
            ('Roles', ', '.join(roles)),
        ]

        await formats.embed(self.bot, entries, title=server.name, author=server.owner, thumbnail=server.icon_url, colour=discord.Colour(0x2ecc71))

    @commands.command(pass_context=True, aliases=["confirm"], hidden=True)
    async def verify(self, ctx, *, member : discord.Member = None):
        if member is None:
            member = ctx.message.author

        tags = await self.bot.db.redis.smembers("user:{0}:tags".format(member.id))

        verify_tag = None

        if "helper" in tags:
            verify_tag = "Helper"

        if "mod" in tags:
            verify_tag = "Moderator"

        if "owner" in tags:
            verify_tag = "owner"

        if verify_tag is not None:
            await self.bot.say("{0} is a {1} at Beemo's HQ.".format(member.mention, verify_tag))

def setup(bot):
    bot.add_cog(Info(bot))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import datetime

def get_daten_fuers_kursheft(startdatum, enddatum, kurstage):
    wochentage = ["Montag", "Dienstag", "Mittwoch",
                  "Donnerstag", "Freitag", "Samstag",
                  "Sonntag"]
    datum = startdatum
    while datum <= enddatum:
        if wochentage[datum.weekday()] in kurstage:
            print(wochentage[datum.weekday()][:2],"., ",
                  datum.strftime('%d.%m.'), " ",
                  kurstage[wochentage[datum.weekday()]],
                  sep="")
        datum += datetime.timedelta(1)

def get_notenstufen(separator=";"):
    for note in range(1,7):
        for vorzeichen in ["+","","-"]:
            if note == 6:
                print(note)
                break
            else:
                print(str(note) + vorzeichen, end=";")

def klassenspiegel_erzeugen(eins,
                            zwei   = 0,
                            drei   = 0,
                            vier   = 0,
                            fuenf  = 0,
                            sechs  = 0,
                            anzahl = None):
    box = '''
Notenspiegel
============

+---+---+---+---+---+---+---+-----+---+
| n | 1 | 2 | 3 | 4 | 5 | 6 |  Ø  | N |
+---+---+---+---+---+---+---+-----+---+
| {:2d}| {:2d}| {:2d}| {:2d}| {:2d}| {:2d}| {:2d}| {:.2f}| {:2d}|
+---+---+---+---+---+---+---+-----+---+

n = Anzahl der Lernenden
N = Nachschrift
    '''
    n  = eins + zwei + drei + vier + fuenf + sechs
    schnitt = (eins + zwei * 2 + drei * 3 + vier * 4 + fuenf * 5 + sechs * 6) / n
    if anzahl is None:
        anzahl = 0
    else:
        anzahl -= n
    print(box.format(n, eins, zwei, drei, vier, fuenf, sechs, schnitt, anzahl))

def kursliste2schuelerliste(kursliste, nurklausur=False):
    """ Wandelt die Schülerdaten einer Kursliste
    in eine „Vornamen Nachnamen“-Liste um und
    gibt sie auf dem Bildschirm aus.

    Bekommt eine wohlgeformte Kursliste der
    folgenden Form als mehrzeiligen String:
        1. NACHNAME, VORNAME WEITERER_VORNAME GKS X

    Der Parameter nurklausur legt fest, ob nur
    die Schüler und Schülerinnen ausgegeben
    werden, die das Fach schriftlich gewählt
    haben.

    Und liefert zurück:
        VORNAME WEITERER_VORNAME NACHNAME

    Bekannte Probleme: Verursacht Probleme,
    wenn der erste Vorname nicht der Haupt-Vorname
    ist und berücksichtigt "von" usw. nicht.
    """
    kursliste = kursliste.split("\n")
    ausgabe = []
    for namenszeile in kursliste:
        nachname, vorname = namenszeile.split(". ")[1].split(", ")
        vorname = vorname.split(" ")[0]
        if nurklausur == False:
            # Gib alles aus
            ausgabe.append(vorname + " " + nachname)
        elif nurklausur and namenszeile.endswith(" X"):
            # Gib die Klausurschreiberinnen
            # und Klausurschreiber aus
            ausgabe.append(vorname + " " + nachname)
    ausgabe_str = "\n".join(ausgabe)
    ausgabe_str = ausgabe_str.replace("  ", " ")
    for line in ausgabe_str.split("\n"):
        print(line)

def kursliste2schuelerliste2(kursliste, nurklausur=False):
    kursliste = kursliste.replace("AB3 X", "_KLAUSUR_").replace("AB4 X", "_KLAUSUR_").replace("GKS X", "_KLAUSUR_").replace("GKM", "")
    for line in kursliste.split("\n"):

        if nurklausur:
            if "_KLAUSUR_" in line:
                line = line.replace("_KLAUSUR_", "")
                nachname, vornamen = line.split(". ")[1].split(", ")
                print(vornamen.strip(" "), nachname.strip(" "))
        else:
            line = line.replace("_KLAUSUR_", "")
            nachname, vornamen = line.split(". ")[1].split(", ")
            print(vornamen.strip(" "), nachname.strip(" "))

def nachname_vorname2vorname_nachname(namensliste):
    # "Nachname, Vorname" --> "Vorname Nachname"
    namensliste = namensliste.split("\n")
    for line in namensliste:
        nachname, vorname = line.split(", ")
        print(vorname.strip(" "), nachname.strip(" "))

if __name__ == "__main__":

    """
    Test der Funktion get_daten_fuers_kursheft()
    ============================================

    Ausgabe aller Tage mit einer Veranstaltung in einem
    Kurs zwischen Start- und Enddatum zwecks Eintragung
    in einem Kursheft.
    """
    print("\n\nTest der Funktion get_daten_fuers_kursheft()")

    startdatum = datetime.date.today()
    enddatum   = datetime.date(2018,2,20)
    kurstage   = {"Montag"   : "ES",
                  "Mittwoch" : "DS"}
    get_daten_fuers_kursheft(startdatum, enddatum, kurstage)

    """
    Test der Funktion get_notenstufen()
    ===================================

    Man erhält eine Liste mit den Notenstufen.
    """
    print("\n\nTest der Funktion get_notenstufen()")
    get_notenstufen(";")

    """
    Test der Funktion klassenspiegel_erzeugen()
    ===========================================

    Die Funktion erzeugt einen Klassenspiegel auf
    Basis der Eingaben und gibt unter N die Zahl der
    Nachschreiberinnen und Nachschreiber an.
    """
    print("\n\nTest der Funktion klassenspiegel_erzeugen()")

    klassenspiegel_erzeugen(1,2,3,3,2,1,12)

    """
    Test der Funktion kursliste2schuelerliste()
    ===========================================

    Die Funktion extrahiert aus der Namensliste einer
    Kursliste (PDF) Vor- und Nachname und gibt die Daten
    aus. Eingestellt werden kann durch Parameter-Übergabe,
    ob nur Klausurschreiberinnen und Klausurschreiber
    ausgegeben werden sollen.

    ID  Name       Vorname  Fach  Klausur
     1  Broderick  Matthew  AB3   X
    """
    print("\n\nTest der Funktion kursliste2schuelerliste()")

    kursliste = """1. Broderick, Matthew AB3 X
    2. Coleman, Dabney Wharton GKM
    3. Wood, John GKS X
    4. Sheedy, Alexandra Elizabeth GKM
    5. Corbin, Leonard Barry AB4 X"""
    kursliste2schuelerliste(kursliste, nurklausur=False)

    """
    Test der Funktion kursliste2schuelerliste2()
    ===========================================

    Wie zuvor, diese Funktion ist etwas robuster und
    hat weniger logische Probleme. Etwaige überzählige
    Vornamen sind von Hand zu entfernen.
    """
    print("\n\nTest der Funktion kursliste2schuelerliste2()")

    kursliste = """1. Broderick, Matthew von AB3 X
    2. Coleman, Dabney Wharton van GKM
    3. Wood, John GKS X
    4. Sheedy, Alexandra Elizabeth GKM
    5. Corbin, Leonard Barry AB4 X"""
    kursliste2schuelerliste2(kursliste, nurklausur=False)

    """
    Test der Funktion nachname_vorname2vorname_nachname()
    =====================================================

    Die Funktion erhält eine Liste der Form "Nachname,
    Vorname" und macht daraus "Vorname Nachname", wobei
    überzählige Vornamen von Hand zu löschen sind.
    """
    print("\n\nTest der Funktion nachname_vorname2vorname_nachname()")

    namensliste = """Broderick, Matthew
                     Coleman, Dabney
                     Wood, John
                     Sheedy, Ally
                     Corbin, Barry"""
    nachname_vorname2vorname_nachname(namensliste)


# -*- coding: utf-8 -*-
# @Time    : 2020/12/1 21:00
# @Author  : zhiming.qian
# @Email   : zhimingqian@tencent.com

import argparse
import os
import torch
from torch.nn.parallel import DataParallel
from torch.nn.parallel.distributed import DistributedDataParallel

from configs import cfg
from mvt.cores.ops import fuse_conv_bn
from mvt.utils.parallel_util import init_dist
from mvt.utils.misc_util import get_dist_info
from mvt.utils.checkpoint_util import load_checkpoint
from mvt.engines.evaluator import multi_device_test, single_device_test
from mvt.datasets.data_builder import build_dataloader, build_dataset
from mvt.models.model_builder import build_model
from mvt.utils.config_util import (
    get_task_cfg,
    get_dataset_global_args,
    get_dict_from_list,
    convert_to_dict,
)
from mvt.utils.io_util import obj_dump

def parse_args():
    parser = argparse.ArgumentParser(description="test (and eval) a model")
    parser.add_argument("task_config", help="test config file")
    parser.add_argument("checkpoint", help="checkpoint file")
    parser.add_argument("--out", help="output result file in pickle format")
    parser.add_argument(
        "--fuse-conv-bn",
        action="store_true",
        help="Whether to fuse conv and bn, this will slightly increase"
        "the inference speed",
    )
    parser.add_argument(
        "--format-only",
        action="store_true",
        help="Format the output results without perform evaluation. It is"
        "useful when you want to format the result to a specific format and "
        "submit it to the test server",
    )
    parser.add_argument(
        "--eval",
        type=str,
        nargs="+",
        help='evaluation metrics, which depends on the dataset, e.g., "bbox",'
        ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC',
    )
    parser.add_argument("--show", action="store_true", help="show results")
    parser.add_argument(
        "--show-dir", help="directory where painted images will be saved"
    )
    parser.add_argument(
        "--show-score-thr",
        type=float,
        default=0.3,
        help="score threshold (default: 0.3)",
    )
    parser.add_argument(
        "--gpu-collect",
        action="store_true",
        help="whether to use gpu to collect results.",
    )
    parser.add_argument(
        "--tmpdir",
        help="tmp directory used for collecting results from multiple "
        "workers, available when gpu-collect is not specified",
    )
    parser.add_argument(
        "--launcher",
        choices=["none", "pytorch", "slurm", "mpi"],
        default="none",
        help="job launcher",
    )
    parser.add_argument("--local_rank", type=int, default=0)
    args = parser.parse_args()
    if "LOCAL_RANK" not in os.environ:
        os.environ["LOCAL_RANK"] = str(args.local_rank)

    return args

def main():
    args = parse_args()

    assert args.out or args.eval or args.format_only or args.show or args.show_dir, (
        "Please specify at least one operation (save/eval/format/show the "
        'results / save the results) with the argument "--out", "--eval"'
        ', "--format-only", "--show" or "--show-dir"'
    )

    if args.eval and args.format_only:
        raise ValueError("--eval and --format_only cannot be both specified")

    if args.out is not None and not args.out.endswith((".pkl", ".pickle")):
        raise ValueError("The output file must be a pkl file.")

    get_task_cfg(cfg, args.task_config)

    # init distributed env first, since logger depends on the dist info.
    if args.launcher == "none":
        distributed = False
    else:
        distributed = True
        if "DIST_PARAMS" in cfg.RUNTIME:
            if isinstance(cfg.RUNTIME.DIST_PARAMS, list):
                init_dist(args.launcher, **get_dict_from_list(cfg.RUNTIME.DIST_PARAMS))
        else:
            init_dist(args.launcher)

    # build the dataloader
    dataset_args = get_dataset_global_args(cfg.DATA)
    dataset = build_dataset(cfg.DATA.VAL_DATA, cfg.DATA.TEST_TRANSFORMS, dataset_args)
    data_loader = build_dataloader(
        dataset,
        samples_per_device=cfg.DATA.VAL_DATA.SAMPLES_PER_DEVICE,
        workers_per_device=cfg.DATA.VAL_DATA.WORKERS_PER_DEVICE,
        dist=distributed,
        shuffle=False,
    )

    # build the model and load checkpoint
    model = build_model(cfg.MODEL)

    print("Loading checkpoint from: ", args.checkpoint)
    checkpoint = load_checkpoint(model, args.checkpoint, map_location="cpu")
    if args.fuse_conv_bn:
        model = fuse_conv_bn(model)

    # this walkaround is for backward compatibility
    if "class_names" in checkpoint["meta"]:
        model.class_names = checkpoint["meta"]["class_names"]
    else:
        if "class_names" in dataset:
            model.class_names = dataset.class_names

    if not distributed:
        model = DataParallel(model, device_ids=[0])
        outputs = single_device_test(
            model,
            data_loader,
            model_type=cfg.MODEL.TYPE,
            show=args.show,
            out_dir=args.show_dir,
            show_score_thr=args.show_score_thr,
        )
    else:
        model = DistributedDataParallel(
            model.cuda(),
            device_ids=[torch.cuda.current_device()],
            broadcast_buffers=False,
        )
        outputs = multi_device_test(
            model,
            data_loader,
            model_type=cfg.MODEL.TYPE,
            tmpdir=args.tmpdir,
            gpu_collect=args.gpu_collect,
        )

    rank, _ = get_dist_info()
    if rank == 0:
        if args.out:
            print(f"\nwriting results to {args.out}")
            obj_dump(outputs, args.out)
        kwargs = {}
        if args.format_only:
            dataset.format_results(outputs, **kwargs)
        if args.eval:
            if len(cfg.RUNTIME.EVALUATION) > 0:
                eval_kwargs = convert_to_dict(cfg.RUNTIME.EVALUATION)
            else:
                eval_kwargs = {}
            # hard-code way to remove EvalHook args
            for key in ["interval", "tmpdir", "start", "gpu_collect"]:
                eval_kwargs.pop(key, None)
            eval_kwargs.update(dict(metric=args.eval, **kwargs))
            print(dataset.evaluate(outputs, **eval_kwargs))

if __name__ == "__main__":
    main()


import asyncio
import requests
import timeit
import threading

def _invoke_api():
    print('thread id:', threading.get_ident())
    url = 'https://data.taipei/api/v1/dataset/cba4734b-3693-46cc-bcc1-61cac76386b9?scope=resourceAquire'
    response = requests.get(url, json={
        'resource_id': 'cba4734b-3693-46cc-bcc1-61cac76386b9'
    }, verify=False)
    print(response.json()['result']['results'][0])

async def sub_function(index: int, loop:asyncio.AbstractEventLoop):
    print(index, 'entry sub function')
    await asyncio.to_thread(_invoke_api)
    # future = loop.run_in_executor(None, _invoke_api)
    # await asyncio.wait_for(future, timeout=3, loop=loop)
    print('leave sub function')

async def main():
    loop = asyncio.get_event_loop()
    # task = loop.create_task(_invoke_api())
    # await asyncio.gather(_invoke_api())
    task = []
    # for i in range(0, 5):
    #     task.append(asyncio.create_task(sub_function(i)))
    task.append(sub_function(1, loop))
    await asyncio.gather(*task)

if __name__ == '__main__':
    start = timeit.default_timer()
    asyncio.run(main())
    end = timeit.default_timer()
    print(start, end, end-start)


from kivy.app import App
from kivy.uix.widget import Widget
from kivy.event import EventDispatcher
from kivy.logger import Logger

class ControlBar(Widget, EventDispatcher):

    def __init__(self, **kwargs):
        super(ControlBar, self).__init__()
        self.app = App.get_running_app()

        # Register events
        self.register_event_type('on_screen_switch')

    def switch_screen(self, instance):
        screen = 'home'
        if instance == self.ids.home_screen_btn:
            screen = 'home'
        elif instance == self.ids.media_screen_btn:
            screen = 'media'
        elif instance == self.ids.phone_screen_btn:
            screen = 'phone'

        self.dispatch('on_screen_switch', screen)

    def on_screen_switch(self, screen, *args):
        pass


from setuptools import find_packages, setup

with open("README.md", "r", encoding="utf-8") as f:
    long_description = f.read()
with open("requirements.txt", "r", encoding="utf-8") as f:
    requirements = f.read()

setup(
    name="create-py-project-by-bhimrazy",
    version="0.1.0",
    author="Bhimraj Yadav",
    author_email="bhimrajyadav977@gmail.com",
    description="A  CLI tool to scaffold python projects. Generate flask, FastAPI, Django, Ml/DL,etc projects using this tool.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/bhimrazy/create-py-project",
    packages=find_packages(),
    py_modules=["main"],
    install_requires=[requirements],
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    entry_points="""
        [console_scripts]
        create-py-project=create_py_project.main:main
    """,
    python_requires=">=3.6",
)


"""
Module that wraps some legacy code to get a set of pod coefficients for
subdomains (domain-decomposed) from a domain decomposed flow past cylinder
problem.

Note this code is meant to be a wrapper for legacy code that is intended to
not be used used very often or in a critical/production setting. Therefore
sustainability  may be lacking.
"""

import vtktools
import numpy as np
from utils import get_grid_end_points
import argparse
import sys

# Backwards compatibility with python 2
if sys.version_info[0] < 3:
    import u2r  # noqa
else:
    import u2rpy3  # noqa
    u2r = u2rpy3

__author__ = " Claire Heaney, Zef Wolffs"
__credits__ = ["Jon Atli Tomasson"]
__license__ = "MIT"
__version__ = "1.0.0"
__maintainer__ = "Zef Wolffs"
__email__ = "zefwolffs@gmail.com"
__status__ = "Development"

def get_pod_coeffs(data_dir='./submodules/DD-GAN/data/FPC_Re3900_2D_CG_old/',
                   data_file_base='fpc_2D_Re3900_CG_',
                   out_dir='./../../data/processed/', nTime=1400,
                   offset=20, field_names=['Velocity'], nGrids=4, xlength=2.2,
                   ylength=0.41, nloc=3, nScalar=2, nDim=2):
    """
    Function that wraps some legacy code to interpolated data from  an
    unstructured mesh to a structured mesh and calculate POD coefficients from
    output.

    Args:
        data_dir (str, optional): Input data folder.
            Defaults to `./submodules/DD-GAN/data/FPC_Re3900_2D_CG_old/`.
        data_file_base (str, optional): Base filename, timesteps will be
            appended. Defaults to `fpc_`.
        out_dir (str, optional): Output data folder. Defaults to
            `./../../data/processed/`.
        nTime (int, optional): Number of timesteps to include in snapshots
            matrix. Defaults to 1400.
        offset (int, optional): At which time level to start taking the
            snapshots. Defaults to 20.
        field_names (list, optional): Names of fields to include from vtu
            data file. Defaults to [`Velocity`].
        nGrids (int, optional): Number of grids of decomposed domain,
            choose 1 or 4. Defaults to 4.
        xlength (float, optional): Length of interpolated domain in x.
            Defaults to 2.2.
        ylength (float, optional): Length of interpolated domain in y.
            Defaults to 0.41.
        nloc (int, optional): Number of local nodes, ie three nodes per
            element (in 2D). Defaults to 3.
        nScalar (int, optional): Dimension of fields. Defaults to 2.
        nDim (int, optional): Dimension of problem. Defaults to 2.
    """

    nFields = len(field_names)

    if nGrids == 4:
        nx = 55
        ny = 42
        nz = 1  # nz = 1 for 2D problems
    elif nGrids == 1:
        nx = 221
        ny = 42
        nz = 1  # nz = 1 for 2D problems
    else:
        print('nx, ny, nz not known for ', nGrids, 'grids')

    grid_origin = [0.0, 0.0]
    grid_width = [xlength/nGrids, 0.0]
    # ddx = np.array((0.01,0.01))
    ddx = np.array((xlength/(nGrids*(nx-1)), ylength/(ny-1)))
    print('ddx', ddx)

    # get a vtu file (any will do as the mesh is not adapted)
    filename = data_dir + data_file_base + '0.vtu'
    representative_vtu = vtktools.vtu(filename)
    coordinates = representative_vtu.GetLocations()

    nNodes = coordinates.shape[0]  # vtu_data.ugrid.GetNumberOfPoints()
    nEl = representative_vtu.ugrid.GetNumberOfCells()
    print('nEl', nEl, type(nEl), 'nNodes', nNodes)
    # nNodes = 3571
    # nEl = 6850

    x_all = np.transpose(coordinates[:, 0:nDim])  # coords n,3  x_all 2,n

    # get global node numbers
    x_ndgln = np.zeros((nEl*nloc), dtype=int)
    for iEl in range(nEl):
        n = representative_vtu.GetCellPoints(iEl) + 1
        x_ndgln[iEl*nloc:(iEl+1)*nloc] = n

    # -------------------------------------------------------------------------------------------------
    # find node duplications when superposing results
    my_field = representative_vtu.GetField(field_names[0])[:, 0]
    my_field = 1
    nScalar_test = 1
    # for one timestep
    # for one field
    value_mesh = np.zeros((nScalar_test, nNodes, 1))  # nTime=1
    value_mesh[:, :, 0] = np.transpose(my_field)
    superposed_grids = np.zeros((nNodes))
    for iGrid in range(nGrids):
        block_x_start = get_grid_end_points(grid_origin, grid_width, iGrid)

        zeros_on_mesh = 0
        value_grid = u2r.simple_interpolate_from_mesh_to_grid(value_mesh,
                                                              x_all, x_ndgln,
                                                              ddx,
                                                              block_x_start,
                                                              nx, ny, nz,
                                                              zeros_on_mesh,
                                                              nEl, nloc,
                                                              nNodes,
                                                              nScalar_test,
                                                              nDim, 1)

        zeros_on_grid = 1
        value_back_on_mesh = u2r.interpolate_from_grid_to_mesh(value_grid,
                                                               block_x_start,
                                                               ddx, x_all,
                                                               zeros_on_grid,
                                                               nScalar_test,
                                                               nx, ny, nz,
                                                               nNodes, nDim, 1)

        superposed_grids = superposed_grids + \
            np.rint(np.squeeze(value_back_on_mesh))

    superposed_grids = np.array(superposed_grids, dtype='int')
    duplicated_nodal_values = []
    for iNode in range(nNodes):
        if superposed_grids[iNode] == 0:
            # this is bad news - the node hasn't appeared in any grid
            print('zero:', iNode)
        elif superposed_grids[iNode] == 2:
            print('two:', iNode)
            # the node appears in two grids - deal with this later
            duplicated_nodal_values.append(iNode)
        elif superposed_grids[iNode] != 1:
            # most of the nodes will appear in one grid
            print('unknown:', iNode, superposed_grids[iNode])

    # -------------------------------------------------------------------------------------------------
    # build up the snapshots matrix from solutions on each of the grids
    snapshots_data = []
    for iField in range(nFields):
        # nDoF = nNodes # could be different value per field
        snapshots_data.append(np.zeros((nx*ny*nz*nDim, nGrids*nTime)))

    # value_mesh = np.zeros((nScalar,nNodes)) # no need to initialise -
    # overwritten
    for iTime in range(nTime):

        # print('')
        # print('time level', iTime)

        filename = data_dir + data_file_base + str(offset+iTime) + '.vtu'
        vtu_data = vtktools.vtu(filename)

        for iField in range(nFields):

            my_field = vtu_data.GetField(field_names[iField])[:, 0:nDim]

            for iGrid in range(nGrids):

                block_x_start = get_grid_end_points(grid_origin, grid_width,
                                                    iGrid)
                if iTime == 0:
                    print('block_x_start', block_x_start)

                # value_mesh = np.zeros((nScalar,nNodes,nTime)) # nTime - this
                # must need initialising here
                # value_mesh[:,:,iTime] = np.transpose(my_field)

                value_mesh = np.transpose(my_field)  # size nScalar,nNodes

                # interpolate field onto structured mesh
                # feed in one result at t time (no need to store in value_mesh
                # in this case)

                # 0 extrapolate solution (for the cylinder in fpc); 1 gives
                # zeros for nodes outside mesh
                zeros_beyond_mesh = 0

                value_grid = \
                    u2r.simple_interpolate_from_mesh_to_grid(value_mesh, x_all,
                                                             x_ndgln, ddx,
                                                             block_x_start, nx,
                                                             ny, nz,
                                                             zeros_beyond_mesh,
                                                             nEl, nloc, nNodes,
                                                             nScalar, nDim, 1)

                snapshots_data[iField][:, iTime*nGrids+iGrid] = \
                    value_grid.reshape(-1)

    # ---------------------------------------------------------------------------------------
    # apply POD to the snapshots
    # some POD truncation settings
    cumulative_tol = 0.99
    nPOD = [nTime]  # len(nPOD) = nFields
    nPOD = [-2]
    nPOD = [10]  # 100 50 10

    bases = []
    singular_values = []

    for iField in range(nFields):

        snapshots_matrix = snapshots_data[iField]
        nrows, ncols = snapshots_matrix.shape

        SSmatrix = np.dot(snapshots_matrix.T, snapshots_matrix)

        print('SSmatrix', SSmatrix.shape)
        eigvalues, v = np.linalg.eigh(SSmatrix)
        eigvalues = eigvalues[::-1]

        # get rid of small negative eigenvalues (there shouldn't be any as the
        # eigenvalues of a real, symmetric matrix are non-negative, but
        # sometimes very small negative values do appear)
        eigvalues[eigvalues < 0] = 0
        s_values = np.sqrt(eigvalues)
        # print('s values', s_values[0:20])

        singular_values.append(s_values)

        cumulative_info = np.zeros(len(eigvalues))
        for j in range(len(eigvalues)):
            if j == 0:
                cumulative_info[j] = eigvalues[j]
            else:
                cumulative_info[j] = cumulative_info[j-1] + eigvalues[j]

        cumulative_info = cumulative_info / cumulative_info[-1]
        nAll = len(eigvalues)

    # if nPOD = -1, use cumulative tolerance
    # if nPOD = -2 use all coefficients (or set nPOD = nTime)
    # if nPOD > 0 use nPOD coefficients as defined by the user

        if nPOD[iField] == -1:
            raise NotImplementedError("Option not yet implemented!")
            # SVD truncation - percentage of information captured or number
            cumulative_tol = nirom_options.compression.cumulative_tol[iField] # noqa
            nPOD_iField = sum(cumulative_info <= cumulative_tol)  # tolerance
            nPOD[iField] = nPOD_iField
        elif nPOD[iField] == -2:
            nPOD_iField = nAll
            nPOD[iField] = nPOD_iField
        else:
            nPOD_iField = nPOD[iField]

        print("retaining", nPOD_iField, "basis functions of a possible",
              len(eigvalues))

        # nDim should be nScalar?
        basis_functions = np.zeros((nx*ny*nz*nDim, nPOD_iField))
        for j in reversed(range(nAll-nPOD_iField, nAll)):
            Av = np.dot(snapshots_matrix, v[:, j])
            basis_functions[:, nAll-j-1] = Av/np.linalg.norm(Av)

        bases.append(basis_functions)

    pod_coeffs = []

    for iField in range(nFields):

        basis = bases[iField]

        snapshots_matrix = snapshots_data[iField]
        print('snapshots_matrix', snapshots_matrix.shape)

        for iGrid in range(nGrids):

            # :,iTime*nGrids+iGrid
            # want solutions in time for a particular grid
            snapshots_per_grid = np.zeros((nx*ny*nz*nDim, nTime))
            for iTime in range(nTime):
                # print('taking snapshots from', iTime*nGrids+iGrid )
                snapshots_per_grid[:, iTime] = snapshots_matrix[:,
                                                                iTime*nGrids
                                                                + iGrid]
            print(snapshots_per_grid)
            pod_coeffs.append(np.dot(basis.T, snapshots_per_grid))
            print(basis.shape)

        np.save(out_dir + "/pod_coeffs_field_{}".format(
            field_names[iField]), np.array(pod_coeffs))
        np.save(out_dir + "/pod_basis_field_{}".format(
            field_names[iField]), np.array(basis))
        np.savetxt(out_dir + "/pod_eigenvalues_field_{}".format(
            field_names[iField]), np.array(eigvalues))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Module that wraps some \
legacy code to interpolate data from  an unstructured mesh to a structured \
mesh and calculate POD coefficients from output. \n\n\
Note that output will be saved under the folder specified by out_dir as \
`pod_coeffs_field_<name of field>`. \
The shape of the output matrix will be (num_subgrids, num_pod_coeffs, \
num_time_levels)")
    parser.add_argument('--data_dir', type=str, nargs='?',
                        default="./submodules/DD-GAN/data/\
FPC_Re3900_2D_CG_old/",
                        help='Input data folder')
    parser.add_argument('--data_file_base', type=str, nargs='?',
                        default="fpc_",
                        help='Base filename')
    parser.add_argument('--out_dir', type=str, nargs='?',
                        default="./../../data/processed/",
                        help='Output data folder')
    parser.add_argument('--nTime', type=int, nargs='?', default=2000,
                        help='Number of timesteps to include in snapshots')
    parser.add_argument('--offset', type=int, nargs='?', default=0,
                        help='At which time level to start taking snapshots')
    parser.add_argument('--field_names', type=list, nargs='?',
                        default=['Velocity'],
                        help='Names of fields to include from vtu data file')
    parser.add_argument('--nGrids', type=int, nargs='?', default=4,
                        help='Number of grids of decomposed domain, 1 or 4')
    parser.add_argument('--xlength', type=float, nargs='?', default=2.2,
                        help='Length of interpolated domain in x')
    parser.add_argument('--ylength', type=float, nargs='?', default=0.41,
                        help='Length of interpolated domain in y')
    parser.add_argument('--nloc', type=int, nargs='?', default=3,
                        help='Number of local nodes')
    parser.add_argument('--nScalar', type=int, nargs='?', default=2,
                        help='Dimension of fields')
    parser.add_argument('--nDim', type=int, nargs='?', default=2,
                        help='Dimension of problem.')
    args = parser.parse_args()

    arg_dict = vars(args)

    get_pod_coeffs(**arg_dict)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import warnings

from sklearn.metrics import confusion_matrix
from sklearn.model_selection import learning_curve
from sklearn.model_selection import cross_val_score
from sklearn.exceptions import ConvergenceWarning

def plot_confusion_matrix(y, y_pred, normalize=True, **kwargs):

    conf_mx = confusion_matrix(y, y_pred)

    if normalize:
        row_sums = conf_mx.sum(axis=1, keepdims=True)
        conf_mx = conf_mx / row_sums

    ax = plt.axes()
    sns.heatmap(conf_mx, annot=True, fmt=".2f", cmap="coolwarm", center=0, ax=ax, **kwargs)
    ax.set_title('Confusion matrix')
    ax.set_ylabel('Observed')
    ax.set_xlabel('Predicted')

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None, scoring=None,
                        n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5)):
    """Generate a simple plot of the test and training learning curve"""
    plt.figure()
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel("Training examples")
    plt.ylabel("Score")
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, scoring=scoring, n_jobs=n_jobs, train_sizes=train_sizes)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="Cross-validation score")

    plt.legend(loc="best")
    return plt

def cv_model(transformer_pipeline, alg, X_train, y_train, cv, scoring, model_results=None):
    alg_name = alg.__name__
    X = transformer_pipeline.fit_transform(X_train)
    y = y_train

    cv_scores = cross_val_score(alg(), X, y, cv=cv, scoring=scoring, n_jobs=-1)
    print(f'{alg_name} CV Score: {round(cv_scores.mean(), 5)} with std: {round(cv_scores.std(), 5)}')

    if model_results is not None:
        model_results = model_results.append(pd.DataFrame({'model': alg_name,
                                                           'cv_mean': cv_scores.mean(),
                                                           'cv_std': cv_scores.std()},
                                                          index=[0]),
                                             ignore_index=True)

        return model_results

def report_cv_scores(results, n_top=3):
    for i in range(1, n_top + 1):
        candidates = np.flatnonzero(results['rank_test_score'] == i)
        for candidate in candidates:
            print("Model with rank: {0}".format(i))
            print("Mean validation score: {0:.3f} (std: {1:.3f})".format(
                  results['mean_test_score'][candidate],
                  results['std_test_score'][candidate]))
            print("Parameters: {0}".format(results['params'][candidate]))
            print("")

def compare_algorithm_cv_scores(transformer_pipeline, algs, X_train, y_train, cv, scoring):
    # Filter out warnings from models
    warnings.filterwarnings('ignore', category=ConvergenceWarning)
    warnings.filterwarnings('ignore', category=DeprecationWarning)
    warnings.filterwarnings('ignore', category=UserWarning)

    model_results = pd.DataFrame(columns=['model', 'cv_mean', 'cv_std'])

    for alg in algs:
        model_results = cv_model(transformer_pipeline, alg, X_train, y_train, cv, scoring, model_results)

    return model_results

def plot_algorithm_cv_scores(model_results):
    model_results.set_index('model', inplace=True)
    model_results['cv_mean'].plot.bar(color='orange', figsize=(8, 6),
                                      yerr=list(model_results['cv_std']),
                                      edgecolor='k', linewidth=2)
    plt.title('Model F1 Score Results');
    plt.ylabel('Mean F1 Score (with error bar)');
    model_results.reset_index(inplace=True)

def plot_feature_importances(df, n=20, threshold=None, return_df=False):
    """
    Slightly adapted from https://www.kaggle.com/willkoehrsen/a-complete-introduction-and-walkthrough
    Plots n most important features. Also plots the cumulative importance if
    threshold is specified and prints the number of features needed to reach threshold cumulative importance.
    Intended for use with any tree-based feature importances.

    Args:
        df (dataframe): Dataframe of feature importances. Columns must be "feature" and "importance".

        n (int): Number of most important features to plot. Default is 15.

        threshold (float): Threshold for cumulative importance plot. If not provided, no plot is made. Default is None.

    Returns:
        df (dataframe): Dataframe ordered by feature importances with a normalized column (sums to 1)
                        and a cumulative importance column

    Note:

        * Normalization in this case means sums to 1.
        * Cumulative importance is calculated by summing features from most to least important
        * A threshold of 0.9 will show the most important features needed to reach 90% of cumulative importance

    """
    plt.style.use('fivethirtyeight')

    # Sort features with most important at the head
    df = df.sort_values('importance', ascending=False).reset_index(drop=True)

    # Normalize the feature importances to add up to one and calculate cumulative importance
    df['importance_normalized'] = df['importance'] / df['importance'].sum()
    df['cumulative_importance'] = np.cumsum(df['importance_normalized'])

    plt.rcParams['font.size'] = 12

    # Bar plot of n most important features
    plt.figure(figsize=(8, 0.5*n))
    df.loc[:n, :].plot.barh(y='importance_normalized',
                            x='feature', color='darkgreen',
                            edgecolor='k', figsize=(12, 8),
                            legend=False, linewidth=2)

    plt.xlabel('Normalized Importance', size=18);
    plt.ylabel('');
    plt.title(f'{n} Most Important Features', size=18)
    plt.gca().invert_yaxis()

    if threshold:
        # Cumulative importance plot
        plt.figure(figsize=(8, 6))
        plt.plot(list(range(len(df))), df['cumulative_importance'], 'b-')
        plt.xlabel('Number of Features', size=16);
        plt.ylabel('Cumulative Importance', size=16);
        plt.title('Cumulative Feature Importance', size=18);

        # Number of features needed for threshold cumulative importance
        # This is the index (will need to add 1 for the actual number)
        importance_index = np.min(np.where(df['cumulative_importance'] > threshold))

        # Add vertical line to plot
        plt.vlines(importance_index + 1, ymin=0, ymax=1.05, linestyles='--', colors='red')
        plt.show();

        print('{} features required for {:.0f}% of cumulative importance.'.format(importance_index + 1,
                                                                                100 * threshold))

    if return_df:
        return df


"""Code that takes an Object schema and outputs a string that describes its schema.

Without fine-tuning the LLM, the quality of the response may end up depending
on details such as the schema description in the prompt.

Users can implement their own type descriptors or customize an existing one
using inheritance and over-loading and provide the type-descriptors to
the create_extraction_chain function.
"""
import abc
from typing import Any, Iterable, List, TypeVar, Union

from kor.nodes import (
    AbstractSchemaNode,
    AbstractVisitor,
    Bool,
    Number,
    Object,
    Selection,
    Text,
)

T = TypeVar("T")

# PUBLIC API

class TypeDescriptor(AbstractVisitor[T], abc.ABC):
    """Abstract interface for a type-descriptor.

    A type-descriptor is responsible for taking in a schema and outputting its type
    as a string. The description is used to help the LLM generate structured output.

    A type-descriptor is a visitor that can be used to traverse the schema recursively.
    """

    @abc.abstractmethod
    def describe(self, node: Object) -> str:
        """Take in node and describe its type as a string."""
        raise NotImplementedError()

class BulletPointDescriptor(TypeDescriptor[Iterable[str]]):
    """Generate a bullet point style schema description."""

    def visit_default(self, node: "AbstractSchemaNode", **kwargs: Any) -> List[str]:
        """Default action for a node."""
        depth = kwargs["depth"]
        space = "* " + depth * " "
        return [f"{space}{node.id}: {node.__class__.__name__} # {node.description}"]

    def visit_object(self, node: Object, **kwargs: Any) -> List[str]:
        """Visit an object node."""
        depth = kwargs["depth"]
        code_lines = self.visit_default(node, depth=depth)
        for child in node.attributes:
            code_lines.extend(child.accept(self, depth=depth + 1))
        return code_lines

    def describe(self, node: Object) -> str:
        """Describe the type of the given node."""
        code_lines = node.accept(self, depth=0)
        return "\n".join(code_lines)

class TypeScriptDescriptor(TypeDescriptor[Iterable[str]]):
    """Generate a typescript style schema description."""

    def visit_default(self, node: "AbstractSchemaNode", **kwargs: Any) -> List[str]:
        """Default action for a node."""
        depth = kwargs["depth"]
        space = depth * " "

        if isinstance(node, Selection):
            finalized_type = " | ".join('"' + s.id + '"' for s in node.options)
        elif isinstance(node, Text):
            finalized_type = "string"
        elif isinstance(node, Number):
            finalized_type = "number"
        elif isinstance(node, Bool):
            finalized_type = "boolean"
        else:
            raise NotImplementedError()

        if node.many:
            finalized_type = "Array<" + finalized_type + ">"

        return [f"{space}{node.id}: {finalized_type} // {node.description}"]

    def visit_object(self, node: Object, **kwargs: Any) -> List[str]:
        """Visit an object node."""
        depth = kwargs["depth"]
        space = depth * " "

        if node.many:
            many_formatter = "Array<"
        else:
            many_formatter = ""

        code_lines = [f"{space}{node.id}: {many_formatter}{{ // {node.description}"]

        for child in node.attributes:
            code_lines.extend(child.accept(self, depth=depth + 1))

        if node.many:
            many_formatter = ">"
        else:
            many_formatter = ""

        code_lines.append(f"{space}}}{many_formatter}")
        return code_lines

    def describe(self, node: "Object") -> str:
        """Describe the node type in TypeScript notation."""
        if not isinstance(node, Object):
            raise TypeError(f"Expecting an Object node got {node}")

        code_lines = node.accept(self, depth=0)
        code = "\n".join(code_lines)
        return f"```TypeScript\n\n{code}\n```\n"

def initialize_type_descriptors(
    type_descriptor: Union[TypeDescriptor, str]
) -> TypeDescriptor:
    """Initialize the type descriptors."""
    if isinstance(type_descriptor, str):
        if type_descriptor == "bullet_point":
            return BulletPointDescriptor()
        elif type_descriptor == "typescript":
            return TypeScriptDescriptor()
        else:
            raise ValueError(
                f"Unknown type descriptor: {type_descriptor}. Use one of: bullet_point,"
                " typescript or else provide an instance of TypeDescriptor."
            )
    return type_descriptor


import errno
import argparse
import inspect
import sys
import traceback
from argparse import RawTextHelpFormatter
from cortx.utils.conf_store import Conf
from cortx.utils.conf_store.error import ConfError
from cortx.utils.schema import Format

class ConfCli:
    """ CLI for the Conf Store """
    _index = "conf_cli"

    @staticmethod
    def init(url: str):
        """ Load ConfStore URL """
        Conf.load(ConfCli._index, url)

    @staticmethod
    def set(args):
        """ Set Key Value """
        kv_delim = '=' if args.kv_delim == None else args.kv_delim
        if len(kv_delim) > 1 or kv_delim not in [':', '>', '.', '|', '/', '=']:
            raise ConfError(errno.EINVAL, "invalid delim %s", kv_delim)
        kv_list = args.args[0].split(';')
        for kv in kv_list:
            try:
                key, val = kv.split(kv_delim, 1)
            except:
               raise ConfError(errno.EINVAL, "Invalid KV pair %s", kv)
            Conf.set(ConfCli._index, key, val)
        Conf.save(ConfCli._index)

    @staticmethod
    def get(args) -> str:
        """ Obtain value for the given keys """
        params = args.args
        key_list = params[0].split(';')
        n_keys = len(key_list)
        def_val_list = list(None for i in range(0, n_keys))
        if len(params) > 1:
            def_val_list = params[1].split(';')
            if len(def_val_list) != n_keys:
                raise ConfError(errno.EINVAL,
                    "No. of default values, dont match no. of keys")
        val_list = []
        for i in range(0, n_keys):
            val = Conf.get(ConfCli._index, key_list[i], def_val_list[i])
            val_list.append(val)
        format_type = 'json' if args.format == None else args.format
        return Format.dump(val_list, format_type)

    @staticmethod
    def delete(args):
        """ Deletes given set of keys from the config """
        key_list = args.args[0].split(';')
        is_deleted = []
        for key in key_list:
            status = Conf.delete(ConfCli._index, key)
            is_deleted.append(status)
        if any(is_deleted):
            Conf.save(ConfCli._index)

class GetCmd:
    """ Get Cmd Structure """

    @staticmethod
    def add_args(sub_parser) -> None:
        s_parser = sub_parser.add_parser('get', help=
            "Retrieves the values for one or more keys\n."
            "Multiple keys are separated using ';'.\n"
            "Example(s): 'k1', 'k1>k2;k3', 'k4[2]>k5', 'k6>k4[2]>k5'\n\n"
            "Example command:\n"
            "# conf json:///tmp/csm.conf get 'k6>k4[2]>k5'\n\n")
        s_parser.set_defaults(func=ConfCli.get)
        s_parser.add_argument('-f', dest='format', help=
                'Output Format json(default), yaml or toml')
        s_parser.add_argument('args', nargs='+', default=[], help='args')

class SetCmd:
    """ Set Cmd Structure """

    @staticmethod
    def add_args(sub_parser) -> None:
        s_parser = sub_parser.add_parser('set', help=
            "This command adds the keys and specified values to the conf\n"
            "store. If the key exists, then old value is overwritten.\n\n"
            "One or more Key Value pair as 'key=val' format.\n"
            "Multiple key value pairs are separated using “;” \n"
            "Examples: 'k1=v1', 'k1=v1;k2=v2', 'k4[2]>k5=v6', 'k6>k4[2]>k5=v3'\n\n"
            "Example command:\n"
            "# conf json:///tmp/csm.conf set 'k1>k2=v1;k3=1'\n\n")
        s_parser.set_defaults(func=ConfCli.set)
        s_parser.add_argument('-d', dest='kv_delim',
            help="Delimiter for k=v (default is '=')")
        s_parser.add_argument('args', nargs='+', default=[], help='args')

class DeleteCmd:
    """ Delete Cmd Structure """

    @staticmethod
    def add_args(sub_parser) -> None:
        s_parser = sub_parser.add_parser('delete', help=
            "This command deletes the keys mentioned in the arguments\n"
            "(and the associated values) from the conf store.\n\n"
            "One or more Key(s). Multiple keys are separated using ';'\n"
            "Example(s): 'k1', 'k1>k2;k3', 'k4[2]>k5', 'k6>k4[2]>k5'\n\n"
            "Example command:\n"
            "# conf json:///tmp/csm.conf delete 'k1>k2;k3'\n\n")
        s_parser.set_defaults(func=ConfCli.delete)
        s_parser.add_argument('args', nargs='+', default=[], help='args')

def main():
    # Setup Parser
    parser = argparse.ArgumentParser(description='Conf Store CLI',
        formatter_class=RawTextHelpFormatter)
    parser.add_argument('url', help='URL for the ConfStore backend')
    sub_parser = parser.add_subparsers(title='command',
        help='represents the action from: set, get, delete\n\n', dest='command')

    # Add Command Parsers
    members = inspect.getmembers(sys.modules[__name__])
    for name, cls in members:
        if name != "Cmd" and name.endswith("Cmd"):
            cls.add_args(sub_parser)

    # Parse and Process Arguments
    try:
        args = parser.parse_args()
        ConfCli.init(args.url)
        out = args.func(args)
        if out is not None and len(out) > 0:
            print(out)
        return 0

    except Exception as e:
        sys.stderr.write("%s\n\n" % str(e))
        sys.stderr.write("%s\n" % traceback.format_exc())
        return errno.EINVAL

if __name__ == "__main__":
    rc = main()
    sys.exit(rc)


#!/usr/bin/env python

import ast
import os
import shelve
import threading
import configparser
import namedtupled
import sys
import traceback

from logging import getLogger
from abc import ABCMeta, abstractmethod

import common.framework.logger as logger
import common.framework.config as config

LOGGER = getLogger(__name__)

class BaseApplication(metaclass=ABCMeta):

    SCRIPT_EXT = ".py"
    CONF_EXT = ".ini"
    GENERAL_CONF_NAME = "general" + CONF_EXT

    RET_NORMAL_END = 0
    RET_KEY_INTERRUPTED_END = 1
    RET_ABNORMAL_END = 100

    def __init__(self, module_name, script_name):
        try:
            self.module_name = module_name
            self.script_name = script_name
            self._prepare_config_dir()
            self.load_config()
            self._prepare_appcache()
            self.validate_config()
            self.__toplevel_logger = None
            self.create_toplevel_logger()
        except Exception as ex:
            print(traceback.format_exc(), file=sys.stderr)
            sys.exit(BaseApplication.RET_ABNORMAL_END)

    @abstractmethod
    def validate_config(self):
        pass

    @abstractmethod
    def setup_resource(self):
        pass

    @abstractmethod
    def setup_application(self):
        pass

    @abstractmethod
    def run_application(self, **args):
        pass

    @abstractmethod
    def teardown_application(self):
        pass

    @abstractmethod
    def teardown_resource(self):
        pass

    def _prepare_appcache(self):
        dbname = os.path.join(config.APPCACHE,
                              os.path.basename(self.script_name))

        self.__appcache = shelve.open(dbname, flag="c", writeback=False)
        self.__appcache_mutex = threading.BoundedSemaphore(value=1)

    def set_cache(self, key, value):
        with self.__appcache_mutex:
            self.__appcache[key] = value

    def del_cache(self, key):
        with self.__appcache_mutex:
            del self.__appcache[key]

    def get_cache(self, key):
        return self.__appcache[key]

    def get_cache_keys(self):
        return self.__appcache.keys()

    def _convert_config_type(self, config):
        if isinstance(config, str):
            return ast.literal_eval(config)
        elif isinstance(config, dict):
            result = dict()
            for (key, value) in config.items():
                result[key] = self._convert_config_type(value)
            return result
        else:
            raise Exception("unknown config type: %s" % type(config))

    def load_config(self):
        specific_config_basename = \
            os.path.basename(self.script_name).\
            replace(BaseApplication.SCRIPT_EXT,
                    BaseApplication.CONF_EXT)

        general_conf = configparser.ConfigParser()
        specific_conf = configparser.ConfigParser()

        general_conf.read(os.path.join(config.CONFIG_DIR,
                                       BaseApplication.GENERAL_CONF_NAME))
        specific_conf.read(os.path.join(config.CONFIG_DIR,
                                        specific_config_basename))

        self.cnfg = namedtupled.map(self._convert_config_type(
            general_conf._sections))

        self.cnfs = namedtupled.map(self._convert_config_type(
            specific_conf._sections))

    def _prepare_config_dir(self):
        for directory in config.DIRECTORIES:
            if not os.path.isdir(directory):
                os.mkdir(directory)

    def create_toplevel_logger(self):
        if self.__toplevel_logger is None:
            self.__toplevel_logger = \
                logger.setup_logger(self.module_name,
                                    self.script_name,
                                    self.cnfg.logging.loglevel,
                                    self.cnfg.logging.rotation_timing,
                                    self.cnfg.logging.backupcount)
        return self.__toplevel_logger

    def start(self, **args):
        retcode = BaseApplication.RET_NORMAL_END
        try:
            LOGGER.info("start application!")

            LOGGER.info("start setup resource")
            self.setup_resource()
            LOGGER.info("end setup resource")

            LOGGER.info("start application setup")
            self.setup_application()
            LOGGER.info("end application setup")

            LOGGER.info("start main routine")
            result = self.run_application(**args)
            LOGGER.info("end main routine")

            LOGGER.info("end application without unexpected error")
            if (type(result) is int):
                retcode = result
        except KeyboardInterrupt:
            LOGGER.warn("keyboard interrupted")
            retcode = BaseApplication.RET_KEY_INTERRUPTED_END
        except Exception as ex:
            LOGGER.error("unexpected exception <%s> occurred" % (str(ex)))
            LOGGER.error(traceback.format_exc())
            retcode = BaseApplication.RET_ABNORMAL_END
        finally:
            LOGGER.info("start cleanup")
            try:
                LOGGER.info("start teardown application")
                self.teardown_application()
                LOGGER.info("end teardown application")
            except Exception as ex:
                LOGGER.warning("unexpected exception <%s> occurred" %
                               str(ex))
            try:
                LOGGER.info("start teardown resource")
                self.teardown_resource()
                LOGGER.info("end teardown resource")
            except Exception as ex:
                LOGGER.warning("unexpected exception <%s> occurred" %
                               str(ex))
            try:
                LOGGER.info("start persistent app cache")
                self.__appcache.close()
                LOGGER.info("end persistent app cache")
            except Exception as ex:
                LOGGER.warning("unexpected exception <%s> occurred" %
                               str(ex))
            LOGGER.info("end cleanup")

        sys.exit(retcode)