import numpy as np
import os
from collections import OrderedDict
from tqdm import tqdm
import torch
import torch.optim as optim
from torch.utils.data import DataLoader
import torch.nn.functional as F
import torch.backends.cudnn as cudnn

from dataset import BrainDataset, FACE_OBJECT, MALE_FEMALE, ARTIFICIAL_NATURAL, CLASSIFY_ALL
from dataset import DATA_TYPE_TRAIN, DATA_TYPE_VALIDATION, DATA_TYPE_TEST
from model import get_combined_model
from logger import Logger
import options
import export
from early_stopping import EarlyStopping
from utils import get_device, fix_state_dict, save_result, save_predictions, get_test_subject_ids, fix_run_seed

def train_epoch(model, device, train_loader, optimizers, epoch, logger):
    model.train()

    threshold = torch.Tensor([0.5]).to(device)

    correct = 0
    count = epoch * len(train_loader)

    pbar = tqdm(total=len(train_loader))

    for batch_idx, sample_batched in enumerate(train_loader):
        data_f = sample_batched['fmri_data']
        data_e = sample_batched['eeg_data']
        label  = sample_batched['label']
        data_f, data_e, label = data_f.to(device), data_e.to(device), label.to(device)

        for optimizer in optimizers:
            optimizer.zero_grad()
        output = model(data_f, data_e)
        loss = F.binary_cross_entropy(output, label)
        loss.backward()
        for optimizer in optimizers:
            optimizer.step()

        result = (output > threshold).float() * 1
        pred = torch.sum(result == label).item()
        correct += pred

        count += 1
        logger.log("f_loss/train", loss.item(), count)

        pbar.update()

    accuracy = 100.0 * correct / len(train_loader.dataset)

    pbar.close()

    logger.log("f_accuracy/train", accuracy, epoch)
    return accuracy

def eval_epoch(model, device, validation_loader, epoch, logger, record_result=False):
    model.eval()
    validation_loss = 0
    correct = 0

    threshold = torch.Tensor([0.5]).to(device)

    if record_result:
        recorded_labels = []
        recorded_preds = []

    with torch.no_grad():
        for sample_batched in validation_loader:
            data_f = sample_batched['fmri_data']
            data_e = sample_batched['eeg_data']
            label  = sample_batched['label']
            data_f, data_e, label = data_f.to(device), data_e.to(device), label.to(device)

            output = model(data_f, data_e)

            if record_result:
                recorded_labels += list(label.cpu().detach().numpy().reshape(-1))
                recorded_preds += list(output.cpu().detach().numpy().reshape(-1))

            validation_loss = F.binary_cross_entropy(output,
                                               label,
                                               reduction='sum').item()
            # Sum up batch loss
            result = (output > threshold).float() * 1
            pred = torch.sum(result == label).item()
            correct += pred

    validation_loss /= len(validation_loader.dataset)

    accuracy = 100.0 * correct / len(validation_loader.dataset)

    if logger != None:
        logger.log("f_loss/validation", validation_loss, epoch)
        logger.log("f_accuracy/validation", accuracy, epoch)

    if record_result:
        return accuracy, (recorded_labels, recorded_preds)
    else:
        return accuracy

def merge_state_dict(part_state_dict, combined_state_dict):
    for name, param in part_state_dict.items():
        if name not in combined_state_dict:
            continue
        combined_state_dict[name].copy_(param)

def load_pretrained_models(model, device, classify_type, fold, fix_weights,
                           preload_fmri_dir, preload_eeg_dir):
    fmri_model_path = "{}/model_ct{}_{}.pt".format(preload_fmri_dir, classify_type, fold)
    eeg_model_path  = "{}/model_ct{}_{}.pt".format(preload_eeg_dir, classify_type, fold)
    combined_state = model.state_dict()

    fmri_state = torch.load(fmri_model_path, map_location=device)
    eeg_state  = torch.load(eeg_model_path,  map_location=device)

    # Remove "module." from the key name in state_dict when training with data_parallel
    fmri_state = fix_state_dict(fmri_state)
    eeg_state = fix_state_dict(fmri_state)

    merge_state_dict(fmri_state, combined_state)
    merge_state_dict(eeg_state,  combined_state)

    model.load_state_dict(combined_state)

    if fix_weights:
        model.fix_preloads()

def train_fold(args, classify_type, fold):
    device, use_cuda = get_device(args.gpu)
    kwargs = {'num_workers': 4, 'pin_memory': True} if use_cuda else {}

    test_subject_ids = get_test_subject_ids(args.test_subjects)

    train_loader = DataLoader(BrainDataset(data_type=DATA_TYPE_TRAIN,
                                           classify_type=classify_type,
                                           data_seed=args.data_seed,
                                           use_fmri=True,
                                           use_eeg=True,
                                           data_dir=args.data_dir,
                                           fmri_frame_type=args.fmri_frame_type,
                                           fmri_offset_tr=args.fmri_offset_tr,
                                           eeg_normalize_type=args.eeg_normalize_type,
                                           eeg_frame_type=args.eeg_frame_type,
                                           eeg_duration_type=args.eeg_duration_type,
                                           use_smooth=args.smooth,
                                           average_trial_size=args.average_trial_size,
                                           average_repeat_size=args.average_repeat_size,
                                           fold=fold,
                                           test_subjects=test_subject_ids,
                                           subjects_per_fold=args.subjects_per_fold,
                                           unmatched=args.unmatched,
                                           debug=args.debug),
                              batch_size=args.batch_size,
                              shuffle=True,
                              **kwargs)
    validation_loader = DataLoader(BrainDataset(data_type=DATA_TYPE_VALIDATION,
                                                classify_type=classify_type,
                                                data_seed=args.data_seed,
                                                use_fmri=True,
                                                use_eeg=True,
                                                data_dir=args.data_dir,
                                                fmri_frame_type=args.fmri_frame_type,
                                                fmri_offset_tr=args.fmri_offset_tr,
                                                eeg_normalize_type=args.eeg_normalize_type,
                                                eeg_frame_type=args.eeg_frame_type,
                                                eeg_duration_type=args.eeg_duration_type,
                                                use_smooth=args.smooth,
                                                average_trial_size=args.average_trial_size,
                                                average_repeat_size=args.average_repeat_size,
                                                fold=fold,
                                                test_subjects=test_subject_ids,
                                                subjects_per_fold=args.subjects_per_fold,
                                                unmatched=args.unmatched,
                                                debug=args.debug),
                             batch_size=args.batch_size,
                             shuffle=True,
                             **kwargs)

    if args.run_seed >= 0:
        # Fix random seeds at runtime
        fix_run_seed(args.run_seed + fold)

    fmri_ch_size = train_loader.dataset.fmri_ch_size

    # TODO: combiend modelはeeg_duration_typeに対応していない
    assert (eeg_duration_type == "normal")

    model = get_combined_model(args.model_type, fmri_ch_size,
                               args.kernel_size,
                               args.level_size,
                               args.level_hidden_size,
                               args.residual,
                               args.combined_hidden_size,
                               args.combined_layer_size,
                               args.parallel,
                               device)

    # Loading of pretrained model
    if args.preload_fmri_dir is not None or args.preload_eeg_dir is not None:
        # If either preload_fmri_dir or preload_eeg_dir is None, prelaod will not be done
        load_pretrained_models(model, device, classify_type, fold, args.fix_preloads,
                               args.preload_fmri_dir, args.preload_eeg_dir)

    lr = args.lr
    weight_decay = args.weight_decay

    lr_eeg = args.lr_eeg
    lr_fmri = args.lr_fmri
    if lr_eeg is None:
        # Use lr if not specified
        lr_eeg = lr
    if lr_fmri is None:
        # Use lr if not specified
        lr_fmri = lr

    weight_decay_eeg = args.weight_decay_eeg
    weight_decay_fmri = args.weight_decay_fmri
    if weight_decay_eeg is None:
        # Use weight_decay if not specified
        weight_decay_eeg = weight_decay
    if weight_decay_fmri is None:
        # Use weight_decay if not specified
        weight_decay_fmri = weight_decay

    optimizer_fc = optim.Adam(model.parameters_fc(),
                              lr=lr,
                              weight_decay=weight_decay)
    optimizer_eeg = optim.Adam(model.parameters_eeg(),
                               lr=lr_eeg,
                               weight_decay=weight_decay_eeg)
    optimizer_fmri = optim.Adam(model.parameters_fmri(),
                                lr=lr_fmri,
                                weight_decay=weight_decay_fmri)

    optimizers = [optimizer_fc, optimizer_eeg, optimizer_fmri]

    save_dir = args.save_dir
    log_path = os.path.join(save_dir, "log/log_ct{}_{}".format(classify_type, fold))
    logger = Logger(log_path)

    #ignore_epochs = 10
    # No ignore epochs for preloaded model
    ignore_epochs = 0

    early_stopping = EarlyStopping(patience=args.patience,
                                   ignore_epochs=ignore_epochs,
                                   save_dir=save_dir,
                                   fold=fold,
                                   classify_type=classify_type,
                                   debug=args.debug)

    if args.epochs == 0:
        # In the case of no training, only save the model
        early_stopping.save(model)

    for epoch in range(args.epochs):
        train_accuracy = train_epoch(model, device, train_loader, optimizers, epoch, logger)
        print('{}: Train: Accuracy: {:.0f}%'.format(epoch, train_accuracy))

        validation_accuracy  = eval_epoch(model, device, validation_loader, epoch, logger)
        print('{}: Validation:  Accuracy: {:.0f}%'.format(epoch, validation_accuracy))

        if args.early_stopping:
            stop = early_stopping.check_stopping(validation_accuracy, train_accuracy, epoch, model)
            if stop:
                print("stop early: Max Validation Accuracy: {:.0f}%".format(
                    early_stopping.max_validation_accuracy))
                break

        logger.flush()

    logger.close()

    if args.early_stopping:
        return early_stopping.accompanied_train_accuracy, early_stopping.max_validation_accuracy
    else:
        return train_accuracy, validation_accuracy

def train_ten_folds(args, classify_type):
    train_accuracies = []
    validation_accuracies = []

    for fold in range(args.fold_size):
        print("train fold: {}".format(fold))
        train_accuracy, validation_accuracy = train_fold(args, classify_type, fold)

        train_accuracies.append(train_accuracy)
        validation_accuracies.append(validation_accuracy)

    train_accurcy_mean = np.mean(train_accuracies)
    train_accurcy_std = np.std(train_accuracies)
    validation_accurcy_mean = np.mean(validation_accuracies)
    validation_accurcy_std = np.std(validation_accuracies)

    train_result_str = ""
    validation_result_str = ""
    for train_accuracy in train_accuracies:
        train_result_str += "{:.2f}, ".format(train_accuracy)
    for validation_accuracy in validation_accuracies:
        validation_result_str += "{:.2f}, ".format(validation_accuracy)

    print(train_result_str)
    print(validation_result_str)

    results = OrderedDict([
        ("train_accurcy_mean",train_accurcy_mean),
        ("validation_accurcy_mean",validation_accurcy_mean),
        ("train_accurcy_std",train_accurcy_std),
        ("validation_accurcy_std",validation_accurcy_std),
        ("train_result",train_result_str),
        ("validation_result",validation_result_str)])

    save_result(args.save_dir, classify_type, results)

def test_fold(args, classify_type, fold):
    device, use_cuda = get_device(args.gpu)
    kwargs = {'num_workers': 4, 'pin_memory': True} if use_cuda else {}

    test_subject_ids = get_test_subject_ids(args.test_subjects)

    test_loader = DataLoader(BrainDataset(data_type=DATA_TYPE_TEST,
                                          classify_type=classify_type,
                                          data_seed=args.data_seed,
                                          use_fmri=True,
                                          use_eeg=True,
                                          data_dir=args.data_dir,
                                          fmri_frame_type=args.fmri_frame_type,
                                          fmri_offset_tr=args.fmri_offset_tr,
                                          eeg_normalize_type=args.eeg_normalize_type,
                                          eeg_frame_type=args.eeg_frame_type,
                                          eeg_duration_type=args.eeg_duration_type,
                                          use_smooth=args.smooth,
                                          average_trial_size=args.average_trial_size,
                                          average_repeat_size=args.average_repeat_size,
                                          fold=fold,
                                          test_subjects=test_subject_ids,
                                          subjects_per_fold=args.subjects_per_fold,
                                          unmatched=args.unmatched,
                                          debug=args.debug),
                             batch_size=args.batch_size,
                             shuffle=False,
                             **kwargs)

    if args.run_seed >= 0:
        # Fix random seeds at runtime
        fix_run_seed(args.run_seed + fold)

    fmri_ch_size = test_loader.dataset.fmri_ch_size

    model = get_combined_model(args.model_type, fmri_ch_size,
                               args.kernel_size,
                               args.level_size,
                               args.level_hidden_size,
                               args.residual,
                               args.combined_hidden_size,
                               args.combined_layer_size,
                               args.parallel,
                               device)

    # No need to load a pretrained model here

    model_path  = "{}/model_ct{}_{}.pt".format(args.save_dir, classify_type, fold)
    state = torch.load(model_path, map_location=device)
    state = fix_state_dict(state)

    model.load_state_dict(state)

    test_accuracy, (recorded_labels, recorded_preds) = eval_epoch(
        model, device, test_loader, 0, None, record_result=True)

    return test_accuracy, (recorded_labels, recorded_preds)

def test_ten_folds(args, classify_type):
    print("start ten fold test: classify_type={}".format(classify_type))

    test_accuracies = []

    for fold in range(args.fold_size):
        print("test fold: {}".format(fold))
        test_accuracy, (recorded_labels, recorded_preds) = test_fold(
            args, classify_type, fold)
        test_accuracies.append(test_accuracy)
        # Save predicted values
        save_predictions(args.save_dir, classify_type, fold, recorded_labels, recorded_preds)

    test_accurcy_mean = np.mean(test_accuracies)
    test_accurcy_std = np.std(test_accuracies)

    test_result_str = ""
    for test_accuracy in test_accuracies:
        test_result_str += "{:.2f}, ".format(test_accuracy)

    print(test_result_str)

    results = OrderedDict([
        ("test_accurcy_mean",test_accurcy_mean),
        ("test_accurcy_std",test_accurcy_std),
        ("test_result",test_result_str)])

    save_result(args.save_dir, classify_type, results, for_test=True)

def main():
    args = options.get_combined_args()
    options.save_args(args)

    if args.test == False:
        # Train
        if args.classify_type == CLASSIFY_ALL:
            train_ten_folds(args, classify_type=FACE_OBJECT)
            train_ten_folds(args, classify_type=MALE_FEMALE)
            train_ten_folds(args, classify_type=ARTIFICIAL_NATURAL)
            title = "exp: {}".format(args.save_dir)
            export.export_results(args.save_dir, title)
        else:
            train_ten_folds(args, classify_type=args.classify_type)
    else:
        # Test
        if args.classify_type == CLASSIFY_ALL:
            test_ten_folds(args, classify_type=FACE_OBJECT)
            test_ten_folds(args, classify_type=MALE_FEMALE)
            test_ten_folds(args, classify_type=ARTIFICIAL_NATURAL)
        else:
            test_ten_folds(args, classify_type=args.classify_type)

if __name__ == '__main__':
    main()


import urllib.request
import html.parser
import json
import os
from bs4 import BeautifulSoup
from bs4 import SoupStrainer
from datetime import datetime

def get_sports_info(sport):
    # Lists
    dates = []
    opponents = []
    locations = []
    times = []
    final = []

    # Bounds checking
    date_len = 0
    time_len = 0

    # Just get the schedule table
    schedule_table = SoupStrainer('table', id='schedule-table')
    url = "http://byucougars.com/schedule/" + sport # + "/2016"
    print(url)
    # Get page and run it through the parser
    page = urllib.request.urlopen(url)
    soup = BeautifulSoup(page, 'html.parser', parse_only=schedule_table)

    for e in soup.find_all('br'):
        e.replace_with('|')

    # dates
    for node in soup.select('.date .regular'):
        dates.append(''.join(node.find_all(text=True)))
    date_len = len(dates)
    # opponents
    for node in soup.select('.opp .highlight'):
        opponents.append(''.join(node.find_all(text=True)))

    # locations
    for node in soup.select('.display-directions'):
        locations.append(''.join(node.find_all(text=True)))

    # times/results
    for node in soup.select('.time'):
        times.append(''.join(node.find_all(text=True)))
        # print(''.join(node.find_all(text=True)))
    time_len = len(times)

    i = 0
    # Make sure volleyball doesn't screw up my time list
    if date_len == (time_len - 1):
        times.pop(0)

    # Organize into a dictionary
    for date in dates:
        game_list = []
        temp_date = datetime.strptime(date, '%b %d, %Y')
        formatted_date = "%s-%s-%s %s:%s:%s" % (temp_date.year, temp_date.month, temp_date.day, temp_date.hour, temp_date.minute, temp_date.second)
        game_list.append(formatted_date)
        game_list.append(opponents[i])
        game_list.append(locations[i])
        game_list.append(times[i])
        # print(game_list)
        final.append(game_list)
        i += 1

    # print(final)
    return final

'''
THIS IS THE MAIN PART OF THE FILE
'''

# open sports json file
dir_path = os.path.dirname(os.path.realpath(__file__))

with open(dir_path+'/byu_sports.json', 'r') as sports_file:
    json_string = sports_file.read()
parsed_json = json.loads(json_string)

# Make a list of the keys for the scraper
sports = []
for key, value in parsed_json.items():
    sports.append(key)

# list for sports
sports_list = {}

# run scraper for each sport
for sport in sports:
    info = get_sports_info(sport)
    sports_list[sport.rstrip()] = info
# Create JSON file
json_file = open(dir_path+'/byu_games.json', 'w')
json_array = json.dump(sports_list, json_file)
json_file.close()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys

reload(sys)
sys.setdefaultencoding('utf-8')
import struct
import os
import time
import math

import numpy as np
import matplotlib
import matplotlib.pyplot as plt

from ROOT import *
from datetime import datetime
from array import array
from rawdata import get_raw
from rawdata import smooth
from peakfinder import find_potential_peaks
from peakrefine import split_S2
from peakrefine import peak_width
from peakrefine import accurate_peaks
from peakrefine import accurate_S1
from peakrefine import accurate_S2

from peakproperty import Entropy
from peakproperty import Uniformity
from peakproperty import integral
from peakproperty import sort_area

from configparser import ConfigParser
from sandp import full_path

cfg = ConfigParser()
# cfg.read('/home/nilab/Processor/SanDP/sandp/config/sandix.ini')
cfg.read(full_path('config/sandix.ini'))

s1width_lower_limit = int(cfg['peaks']['s1width_lower_limit'])
s1width_upper_limit = int(cfg['peaks']['s1width_upper_limit'])
s2width_lower_limit = int(cfg['peaks']['s2width_lower_limit'])
s2width_upper_limit = int(cfg['peaks']['s2width_upper_limit'])

nsamps = int(cfg['peaks']['nsamps'])
nsamp_base = int(cfg['peaks']['nsamp_base'])
s1_thre_base = int(cfg['peaks']['s1_thre_base'])
s2_thre_base = int(cfg['peaks']['s2_thre_base'])
trigger_position = int(cfg['peaks']['trigger_position'])

PMTgain = [float(cfg['gains']['ch0_gain']),
           float(cfg['gains']['ch1_gain']),
           float(cfg['gains']['ch2_gain']),
           float(cfg['gains']['ch3_gain'])]

def drawWF(evt, fname, savepath=False):
    dat_raw, channels, MicroSec = get_raw(evt, fname)
    dat_smooth = smooth(dat_raw)

    # ----------->
    rawbaseline = np.mean(dat_raw[0:nsamp_base])
    dat_raw_temp = []
    for i in range(len(dat_raw)):  # TODO: simply it
        dat_raw_temp.append(rawbaseline - dat_raw[i])
    # ----------->

    ## Baseline:
    BaseLineSumSigma = array('f', [0.0])
    BaseLineSumSigma[0] = np.std(dat_raw_temp[:nsamp_base])
    ##=== find potential S1 S2 =====
    S1_potential = find_potential_peaks(dat_smooth[:],
                                        s1width_lower_limit,
                                        s1width_upper_limit,
                                        max(0.001, s1_thre_base * BaseLineSumSigma[0]))

    S2_potential = find_potential_peaks(dat_smooth,
                                        s2width_lower_limit,
                                        s2width_upper_limit,
                                        max(0.001, s2_thre_base * BaseLineSumSigma[0]))
    # print('S1 edges: ', S1_potential)
    # print('S2 edges: ', S2_potential)
    print('Event number: ', evt)
    print('Potential S1 number: ', len(S1_potential))
    # print('Potential S2 number: ', len(S2_potential))

    S2_split = split_S2(dat_smooth, S2_potential, 0.1, 1. / 5)
    # print('S2_split', S2_split)
    S1, S2 = accurate_peaks(dat_smooth, S1_potential, S2_split, s1width_upper_limit)
    # print('accurate_peaks')
    # print('S1: ', S1)
    # print('S2: ', S2)
    S1, S2_temp = accurate_S1(dat_smooth, S1, S2, s1width_upper_limit, nearestS1=400, distanceS1=40)
    # print('s1width_upper_limit', s1width_upper_limit)
    S2 += S2_temp
    # print('accurate_S1')
    # print('S1: ', S1)
    # print('S2: ', S2)
    S2 = accurate_S2(S2)
    # print('accurate_S2')
    # print('S1: ', S1)
    # print('S2: ', S2)

    # print('S1 edges:', S1)
    # print('S2 edges:', S2)
    # print("S1 number: %d" %len(S1))
    # print("S2 number: %d" %len(S2))

    ################ Find the Largest S1 & S2 ======>>>
    # ------------------------------------------------------------------------------------>
    # ------------------------------------------------------------------------------------>
    ## Number of S1 and S2 peaks:
    NbS1Peaks = array('i', [0])
    NbS2Peaks = array('i', [0])
    NbS1Peaks[0] = len(S1)
    NbS2Peaks[0] = len(S2)

    if NbS1Peaks[0] > 100:
        NbS1Peaks[0] = 100
    if NbS2Peaks[0] > 100:
        NbS2Peaks[0] = 100

    ## Baseline for each channel:
    nchannels = array("i", [4])
    BaseLineChannel = array("f", nchannels[0] * [0.0])
    BaseLineChannelSigma = array("f", nchannels[0] * [0.0])

    for i in range(len(channels)):
        BaseLineChannel[i] = np.mean(channels[i][:nsamp_base])
        BaseLineChannelSigma[i] = np.std(channels[i][:nsamp_base])

    ## Peak Area to Generate S1 and S2 sort index：
    S1s = []
    S2s = []
    for i in range(len(channels)):
        S1s.append(integral(S1, channels[i], BaseLineChannel[i], PMTgain[i]))
        S2s.append(integral(S2, channels[i], BaseLineChannel[i], PMTgain[i]))

    S1sTot_tmp = [0] * len(S1)
    S2sTot_tmp = [0] * len(S2)

    for i in range(len(S2)):
        for j in range(len(channels)):
            S2sTot_tmp[i] += S2s[j][i]
    for i in range(len(S1)):
        for j in range(len(channels)):
            S1sTot_tmp[i] += S1s[j][i]

    # print s2 size
    # print('S2 area: ', S2sTot_tmp)

    ## S1 and S2 sort index:
    S1s_Key = sort_area(S1sTot_tmp)
    S2s_Key = sort_area(S2sTot_tmp)

    # print s2 size
    # print('S2 area: ', S2sTot_tmp)

    ## Defining S2sTot and S1sTot:
    maxpeaks = 100
    S1sTot, S2sTot = array("f", maxpeaks * [0.0]), array("f", maxpeaks * [0.0])
    for i in range(NbS1Peaks[0]):
        S1sTot[i] = S1sTot_tmp[S1s_Key[i]]

    for i in range(NbS2Peaks[0]):
        S2sTot[i] = S2sTot_tmp[S2s_Key[i]]

    S1sPeak, S2sPeak = array("f", maxpeaks * [0.0]), array("f", maxpeaks * [0.0])
    for i in range(NbS1Peaks[0]):
        peak = peak_width(dat_smooth, 0.5, S1[S1s_Key[i]])
        S1sPeak[i] = peak[1]

    peaklow = []
    for i in range(NbS2Peaks[0]):
        peak = peak_width(dat_smooth, 0.5, S2[S2s_Key[i]])
        S2sPeak[i] = peak[1]

        peaklow.append(peak_width(dat_smooth, 0.1, S2[S2s_Key[i]]))
        # S2sLowWidth[i] = peaklow[2]-peaklow[0]

    ######################################################################################################:
    ##############################     Make Drawings      ################################################:
    ######################################################################################################:
    samp_len = nsamps  ## from sample to micro-second
    sams = np.linspace(0, samp_len, samp_len)

    miny = min(min(channels[0]), min(channels[1]), min(channels[2]), min(channels[3]))
    maxy = max(max(channels[0]), max(channels[1]), max(channels[2]), max(channels[3]))
    ## individul WF:
    fig_individual = plt.figure(figsize=(8, 7))
    plt.subplot(411)

    plt.xlim(-10, samp_len + 10)
    plt.ylim(miny, maxy)

    plt.xlabel('', fontsize=0)
    plt.ylabel('ch0 [V]', fontsize=10)
    plt.grid(True)
    plt.xticks(fontsize=10)
    plt.yticks(fontsize=10)
    # print(len(sams), len(channels[0]))
    plt.plot(sams, channels[0], color='black', linestyle='-', linewidth=.5, label='')
    # plt.axvline(6500, color='magenta', linestyle='--', linewidth = 1)
    # ------------------------------------------------------------------------------------>
    plt.subplot(412)

    plt.xlim(-10, samp_len + 10)
    plt.ylim(miny, maxy)

    plt.xlabel('Samples [4ns]', fontsize=10)
    plt.ylabel('ch1 [V]', fontsize=10)
    plt.grid(True)
    plt.xticks(fontsize=10)
    plt.yticks(fontsize=10)
    plt.plot(sams, channels[1], color='black', linestyle='-', linewidth=.5, label='')
    # plt.axvline(6500, color='magenta', linestyle='--', linewidth = 1)
    # ------------------------------------------------------------------------------------>

    plt.subplot(413)

    plt.xlim(-10, samp_len + 10)
    plt.ylim(miny, maxy)

    plt.xlabel('Samples [4ns]', fontsize=10)
    plt.ylabel('ch2 [V]', fontsize=10)
    plt.grid(True)
    plt.xticks(fontsize=10)
    plt.yticks(fontsize=10)
    plt.plot(sams, channels[2], color='black', linestyle='-', linewidth=.5, label='')
    # plt.axvline(6500, color='magenta', linestyle='--', linewidth = 1)
    # ------------------------------------------------------------------------------------>
    plt.subplot(414)

    plt.xlim(-10, samp_len + 10)
    plt.ylim(miny, maxy)

    plt.xlabel('Samples [4ns]', fontsize=10)
    plt.ylabel('ch3 [V]', fontsize=10)
    plt.grid(True)
    plt.xticks(fontsize=10)
    plt.yticks(fontsize=10)
    plt.plot(sams, channels[3], color='black', linestyle='-', linewidth=.5, label='')
    # plt.axvline(6500, color='magenta', linestyle='--', linewidth = 1)
    # ------------------------------------------------------------------------------------>
    ######################################################################################################:
    ## Summed WF:
    fig, ax = plt.subplots(figsize=(15, 7))
    plt.xlim(0, samp_len)
    ## plt.xlim(4900,  5100)
    ## plt.ylim(0,  0.5)
    plt.xlabel('Samples', fontsize=15)
    plt.ylabel('Amp [V]', fontsize=15)
    plt.grid(True)
    plt.xticks(fontsize=15)
    plt.yticks(fontsize=15)
    plt.plot(sams, dat_raw_temp, color='black', linestyle='-', linewidth=0.5, label='Summed Waveform')
    plt.plot(sams, dat_smooth, color='springgreen', linestyle='-', linewidth=1., alpha=1, label='Smoothed Waveform')

    ## plot baseline:
    plt.axhline(s1_thre_base * BaseLineSumSigma[0], color='magenta', linestyle='--', linewidth=1., label='S1 Threshold')
    plt.axhline(s2_thre_base * BaseLineSumSigma[0], color='yellow', linestyle='--', linewidth=1., label='S2 Threshold')
    ## S1:
    if len(S1) > 0:
        ## Marker all S1s ===>
        for p1 in range(len(S1)):
            if p1 == 0:
                ax.axvspan(S1[p1][0], S1[p1][1], alpha=0.4, color='magenta', label='S1')
            else:
                ax.axvspan(S1[p1][0], S1[p1][1], alpha=0.4, color='magenta', label='')
                ## Marker the Largest S1 ===>
        plt.text(S1sPeak[0] + 200, dat_smooth[int(S1sPeak[0])], 'S1[0]\n\n%.1f PE' % (S1sTot[0]), fontsize=10,
                 color='blue', fontweight='bold', zorder=100)

    ## S2:
    if len(S2) > 0:
        ## Marker all S2s
        for p2 in range(len(S2)):
            if p2 == 0:
                # ax.axvspan(peaklow[p2][0], peaklow[p2][2], alpha=0.4, facecolor='magenta', edgecolor = 'black', label = 'S2')
                ax.axvspan(S2[p2][0], S2[p2][1], alpha=0.4, facecolor='yellow', edgecolor='black', label='S2')
                # print ('========> ',peaklow[p2][0], peaklow[p2][2])
            else:
                ax.axvspan(S2[p2][0], S2[p2][1], alpha=0.4, facecolor='yellow', edgecolor='black', label='')
                # print ('========> ',peaklow[p2][0], peaklow[p2][2])
        ## Marker the Largest S2 ===>
        plt.text(S2sPeak[0] + 200, dat_smooth[int(S2sPeak[0])], 'S2[0]\n\n%.1f PE' % (S2sTot[0]), fontsize=10,
                 color='blue', fontweight='bold', zorder=100)

    ## Test:
    ## print 'Max smooth: ',np.max(dat_smooth)

    plt.title(fname[-19:-4] + ' event ' + str(evt))
    plt.legend(loc='best', shadow=True, fontsize=12.5)
    plt.ylim(0, )
    if savepath:
        name = fname[-26:-4] + '_event_' + str(evt) + '.png'
        plt.savefig(os.path.join(savepath, name), dpi=300)
        plt.close()
    else:
        plt.show(block=False)
        raw_input("Hit Enter To Close")
        plt.close()


from django.db import DEFAULT_DB_ALIAS
from django.contrib.admin.utils import NestedObjects
from drf_spectacular.utils import extend_schema
from drf_yasg.utils import swagger_auto_schema
from rest_framework.decorators import action
from rest_framework import status
from rest_framework.authtoken.models import Token
from dojo.api_v2 import serializers
from dojo.models import Question, Answer
import itertools

class DeletePreviewModelMixin:
    @extend_schema(
        methods=["GET"],
        responses={
            status.HTTP_200_OK: serializers.DeletePreviewSerializer(many=True)
        },
    )
    @swagger_auto_schema(
        method="get",
        responses={"default": serializers.DeletePreviewSerializer(many=True)},
    )
    @action(detail=True, methods=["get"], filter_backends=[], suffix="List")
    def delete_preview(self, request, pk=None):
        object = self.get_object()

        collector = NestedObjects(using=DEFAULT_DB_ALIAS)
        collector.collect([object])
        rels = collector.nested()

        def flatten(elem):
            if isinstance(elem, list):
                return itertools.chain.from_iterable(map(flatten, elem))
            else:
                return [elem]

        rels = [
            {
                "model": type(x).__name__,
                "id": x.id if hasattr(x, "id") else None,
                "name": str(x)
                if not isinstance(x, Token)
                else "<APITokenIsHidden>",
            }
            for x in flatten(rels)
        ]

        page = self.paginate_queryset(rels)

        serializer = serializers.DeletePreviewSerializer(page, many=True)
        return self.get_paginated_response(serializer.data)

class QuestionSubClassFieldsMixin(object):
    def get_queryset(self):
        return Question.objects.select_subclasses()

class AnswerSubClassFieldsMixin(object):
    def get_queryset(self):
        return Answer.objects.select_subclasses()


from bs4 import BeautifulSoup
import requests
import pandas as pd

data = []
for i in range(200):
    # url of arXiv web page
    url = "https://arxiv.org/search/?query=cs.LG&searchtype=all&order=-announced_date_first&size=50&abstracts=show&start="

    # get html content
    r = requests.get(url + str(i)) # range(200)
    html_content = r.content

    soup = BeautifulSoup(html_content,'html.parser')

    spans = soup.find_all('span',class_ = 'abstract-full has-text-grey-dark mathjax')
    titles = soup.find_all('p',class_ = "title is-5 mathjax")

    for j in range(50):
        data.append([titles[j].get_text().split('\n')[2].strip(),' '.join(spans[j].get_text().strip().split(' ')[:-2])])

df_ = pd.DataFrame(data,columns=['Title','Abstract'])
df_.to_csv("DataBase.csv")


import tkinter as tk
from tkinter import ttk
import tkcalendar as tkc

root = tk.Tk()

#Task Description section
task_description_frame = tk.Frame(root)
task_description_frame.pack()

task_label = tk.Label(task_description_frame, text='Task Description')
task_label.pack(side=tk.LEFT)

task_description = tk.Text(task_description_frame, height=10, width=50)
task_description.pack(side=tk.LEFT)

scroll_text = tk.Scrollbar(task_description_frame)
scroll_text.pack(side=tk.RIGHT, fill=tk.Y)
scroll_text.config(command=task_description.yview)
task_description.config(yscrollcommand=scroll_text.set)

#Assign user section
assign_user_frame = tk.Frame(root)
assign_user_frame.pack()

user_label = tk.Label(assign_user_frame, text="Choose user to assign: ")
user_label.pack(side=tk.LEFT)

user_combobox = ttk.Combobox(assign_user_frame, values=["User1", "User2", "User3", "User4", "User5"])
user_combobox.pack(side=tk.LEFT)

#Closure date section
calendar_frame = tk.Frame(root)
calendar_frame.pack()

closure_date_label = tk.Label(calendar_frame, text="Choose closure date")
closure_date_label.pack(side=tk.LEFT)

closure_date_picker = tkc.Calendar(calendar_frame, selectmode="day", year=2020, month=5, day=22)
closure_date_picker.pack(side=tk.LEFT)

closure_date_btn = tk.Button(calendar_frame, text="Submit date")
closure_date_btn.pack(side=tk.BOTTOM)

#Submit task section
submit_frame = tk.Frame(root)
submit_frame.pack()

submit_btn = tk.Button(submit_frame, text="Submit task")
submit_btn.pack(side=tk.LEFT)

cancel_btn = tk.Button(submit_frame, text="Cancel")
cancel_btn.pack(side=tk.RIGHT)

#Task list section
task_frame = tk.Frame(root)
task_frame.pack()

created_task_label = tk.Label(task_frame, text="Your created tasks")
created_task_label.pack(side=tk.TOP)

created_tasks = tk.Listbox(task_frame, selectmode=tk.MULTIPLE)
created_tasks.pack(side=tk.LEFT)

assign_task_label = tk.Label(task_frame, text="Your assigned task")
assign_task_label.pack(side=tk.TOP)

assinged_tasks = tk.Listbox(task_frame, selectmode=tk.MULTIPLE)
assinged_tasks.pack(side=tk.RIGHT)

root.mainloop()


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^cart/$',views.cart),
    url(r'^detail/$',views.detail),
    url(r'^index/$',views.index),
    url(r'^list/$',views.list),
    url(r'^login/$',views.login),
    url(r'^place_order/$',views.place_order),
    url(r'^register/$',views.register),
    url(r'^user_center_infor/$',views.user_center_info),
    url(r'^user_center_order/$',views.user_center_order),
    url(r'^user_center_site/$',views.user_center_site),
]


import pandas as pd
from sklearn import model_selection

if __name__=="__main__":
    data_dir = '../input/'

    df = pd.read_csv(f"{data_dir}mnist_train.csv")

    df['kfold'] = -1

    y = df.label.values

    kf = model_selection.StratifiedKFold(n_splits=5)

    for f, (t,v) in enumerate(kf.split(X=df,y=y)):
        df.loc[v,'kfold'] = f

    df.to_csv(f"{data_dir}mnist_train_folds.csv")
    print(df.head())
    print(df.kfold.value_counts())


import os
from conan import ConanFile
from conan.tools.cmake import CMakeToolchain, CMake
from conan.tools.files import apply_conandata_patches, rmdir
from conan.tools.system.package_manager import Apt
from conans.tools import get, remove_files_by_mask

class VtkConan(ConanFile):
    name = "vtk"
    version = "9.1.0"
    homepage = "https://www.vtk.org/"

    # Alternative method: can use git directly - helpful when hacking VTK
    git_url = "https://gitlab.kitware.com/vtk/vtk.git"
    # git_url = "/build/git-mirrors/vtk.git"
    git_hash = "v" + version

    url = "https://github.com/paulharris/conan-vtk"
    description = "The Visualization Toolkit (VTK) is an open-source, \
        freely available software system for 3D computer graphics, \
        image processing, and visualization."
    # NO SUBSOURCE # # exports = ["LICENSE.md", "CMakeLists.txt", "FindVTK.cmake"]
    exports = ["CMakeLists.txt"]
    exports_sources = "patches/**"

    vtk_source_folder = "vtk_source"

    settings = "os", "compiler", "build_type", "arch"

    ############
    # list of all possible 3rd parties (as of 9.1.0)
    # with each one marked as where they should come from.
    #
    # cgns:             TODO
    # cli11:            TODO
    # diy2:             TODO
    # doubleconversion: conan (double-conversion)
    # eigen:            conan (eigen)
    # exodusII:         TODO
    # expat:            TODO
    # exprtk:           conan (exprtk)
    # fides:            TODO
    # fmt:              conan (fmt)
    # freetype:         conan (freetype)
    # gl2ps:            internal (not available in CCI - TODO)
    # glew:             conan (glew)
    # h5part:           TODO
    # hdf5:             conan (hdf5) - TODO requires MPI and other things
    # ioss:             TODO
    # jpeg:             TODO
    # jsoncpp:          conan (jsoncpp)
    # kissfft:          TODO
    # libharu:          TODO
    # libproj:          conan (proj)
    # libxml2:          TODO
    # loguru:           TODO
    # lz4:              conan (lz4)
    # lzma:             conan (xz_utils)
    # mpi4py:           TODO
    # netcdf:           TODO
    # ogg:              TODO
    # pegtl:            TODO
    # png:              TODO
    # pugixml:          conan (pugixml)
    # sqlite:           conan (sqlite3)
    # theora:           TODO
    # tiff:             TODO
    # utf8:             conan (utfcpp)
    # verdict:          TODO
    # vpic:             TODO
    # vtkm:             TODO
    # xdmf2:            TODO
    # xdmf3:            TODO
    # zfp:              TODO
    # zlib:             conan
    #
    ############

    generators = "CMakeDeps"

    options = {
              "shared": [True, False]
            , "fPIC": [True, False]

            , "download_vtk_git": [True, False]

            , "use_64bit_ids": ["Auto", True, False]

            , "qt_version": ["Auto", "5", "6"]
            , "qt":      [True, False]
            , "qtquick": [True, False]
            , "qtsql":   [True, False]

            , "imaging":    [True, False]
            , "mpi":        [True, False]
            , "rendering":  [True, False]
            , "standAlone": [True, False]
            , "views":      [True, False]
            , "web":        [True, False]

            # I don't use these so I can't make these ones below
            # , "mpi": [True, False]
            # , "minimal": [True, False]
            # , "ioxml": [True, False]
            # , "mpi_minimal": [True, False]
            }

    default_options = {
              "shared": True
            , "fPIC": True

            , "download_vtk_git": False # use the tarball

            , "use_64bit_ids": "Auto" # TODO normally 32 bit on 32 bit platforms

            , "qt_version": "Auto"
            , "qt": False
            , "qtquick": False
            , "qtsql": False

            , "imaging":    False
            , "mpi":        False
            , "rendering":  False
            , "standAlone": False
            , "views":      False
            , "web":        False

            # TODO try supporting more modules, I chose hdf5 randomly...
            # HDF5 requires 'parallel' to be enabled, which also brings in the MPI requirements.
            # HDF5 is expected to have "parallel" enabled
            # , "hdf5:parallel":   True
            # , "hdf5:enable_cxx": False  # can't be enabled with parallel

            # I don't use these so I can't make these ones below
            # , "mpi": False
            # , "minimal": False
            # , "ioxml": False
            # , "mpi_minimal": False
            }

    no_copy_source = True

    def source(self):
        if self.options.download_vtk_git:
            self.run("git clone -b release --single-branch " + self.git_url + " " + self.vtk_source_folder)
            # note: we give the branch a name so we don't have detached heads
            # TODO change to standard git and python chdir
            self.run("cd " + self.vtk_source_folder + " && git checkout -b branch-" + self.git_hash + " " + self.git_hash)
        else:
            get(**self.conan_data["sources"][self.version],
                    strip_root=True,
                    destination=self.vtk_source_folder,
                    )

        # Note that VTK's cmake will insert its own CMAKE_MODULE_PATH at the
        # front of the list.  This is ok as long as there is nothing in that
        # path that will be found before our conan cmake files...
        # That is why we delete the third party finders.
        # Else, we have to patch VTK's CMakeLists.txt, like so:
        #####
        # diff --git a/CMakeLists.txt b/CMakeLists.txt
        # index c15890cfdc..022f704d75 100644
        # --- a/CMakeLists.txt
        # +++ b/CMakeLists.txt
        # @@ -7,7 +7,7 @@ if (POLICY CMP0127)
        #  endif ()
        #
        #  set(vtk_cmake_dir "${VTK_SOURCE_DIR}/CMake")
        # -list(INSERT CMAKE_MODULE_PATH 0 "${vtk_cmake_dir}")
        # +list(APPEND CMAKE_MODULE_PATH "${vtk_cmake_dir}")
        #####

        # DELETE any of the third party finders, before we build - see note above
        remove_files_by_mask(os.path.join(self.vtk_source_folder,"CMake"), "Find*.cmake")

        # Delete VTK's cmake patches (these support older cmake programs).
        # We do not have to support old cmake: we require an updated cmake instead.
        rmdir(self, os.path.join(self.vtk_source_folder,"CMake","patches"))

    def _third_party(self):
        parties = {
                # LEFT field:  target name for linking, will be used as TARGET::TARGET in package_info()
                # RIGHT field: package/version to require
                  "double-conversion": "double-conversion/3.2.0"
                , "eigen":             "eigen/3.4.0"
                , "exprtk":            "exprtk/0.0.1"
                , "fmt":               "fmt/8.1.1"       # 9.1.0 release docs mention a PR - confirmed merged 8.1.0
                , "freetype":          "freetype/2.11.1" # ... what? ... WORKED with 2022-05-06-freetype
                , "glew":              "glew/2.2.0"
                , "jsoncpp":           "jsoncpp/1.9.5"
                # , "libharu": "libharu/2.3.0" -- use VTK's bundled version - heavily patched
                , "lz4":               "lz4/1.9.3"
                , "proj":              "proj/9.0.0" # if MAJOR version changes, update ThirdParty/libproj/CMakeLists.txt
                , "pugixml":           "pugixml/1.12.1"
                , "sqlite3":           "sqlite3/3.38.1"
                , "utfcpp":            "utfcpp/3.2.1"
                , "xz_utils":          "xz_utils/5.2.5" # VTK calls this lzma
                }

        # TODO figure out how we want to support modules...
        # VTK already has an extensive module system, we would want to mirror or use it
        # if self.options.module_hdf5:
        # parties["hdf5"] = "hdf5/1.12.1"

        if self.options.qt:
            parties["qt"] = "qt/6.2.4"

            # NOTE: could also try QT's offical QT
            # self.requires("qtbase/6.2.4@qt/everywhere")

        return parties

    def requirements(self):
        for pack in self._third_party().values():
            self.requires(pack)

    def build_requirements(self):
        if self.options.rendering:
            Apt(self).install([
                # TODO check these are the packages we still need
                "freeglut3-dev",
                "mesa-common-dev",
                "mesa-utils-extra",
                "libgl1-mesa-dev",
                "libglapi-mesa",
                "libsm-dev",
                "libx11-dev",
                "libxext-dev",
                "libxt-dev",
                "libglu1-mesa-dev"
                ])
            # TODO people on other platforms please
            # contribute what packages are required

    def configure(self):
        if self.settings.compiler == "Visual Studio":
            del self.options.fPIC

    def generate(self):
        tc = CMakeToolchain(self)

        # 64 / 32 bit IDs
        if self.options.use_64bit_ids != "Auto":
            tc.variables["VTK_USE_64BIT_IDS"] = self.options.use_64bit_ids

        # Be sure to set this, otherwise vtkCompilerChecks.cmake will downgrade our CXX standard to 11
        tc.variables["VTK_IGNORE_CMAKE_CXX11_CHECKS"] = "ON"

        # no need for versions on installed names?
        tc.variables["VTK_VERSIONED_INSTALL"] = "OFF"

        # TODO future-proofing
        # tc.variables["VTK_LEGACY_REMOVE"] = "ON" # disable legacy APIs
        # tc.variables["VTK_USE_FUTURE_CONST"] = "ON" # use the newer const-correct APIs

        # TODO development debugging
        # tc.variables["VTK_DEBUG_LEAKS"] = "ON" # use the newer const-correct APIs

        # ON or OFF
        tc.variables["BUILD_TESTING"] = "OFF"
        tc.variables["BUILD_EXAMPLES"] = "OFF"
        tc.variables["BUILD_DOCUMENTATION"] = "OFF"

        # Enable KITs - Quote: "Compiles VTK into a smaller set of libraries."
        # Quote: "Can be useful on platforms where VTK takes a long time to launch due to expensive disk access."
        tc.variables["VTK_ENABLE_KITS"] = "ON"

        #### CUDA / MPI / MEMKIND ####
        # ON or OFF
        tc.variables["VTK_USE_CUDA"]    = "OFF"
        tc.variables["VTK_USE_MEMKIND"] = "OFF"
        tc.variables["VTK_USE_MPI"]     = "OFF"

        # There are LOTS of these modules now ...
        # for vtkModule in self.required_modules:
            # tc.variables["VTK_MODULE_ENABLE_VTK_" + vtkModule] = "YES" or NO or WANT or DONT_WANT

        # TODO support more modules - see notes above re hdf5
        # tc.variables["VTK_MODULE_ENABLE_VTK_hdf5"] = "YES" # or NO or WANT or DONT_WANT

        # https://gitlab.kitware.com/vtk/vtk/-/blob/master/Documentation/dev/build.md
        # TODO try VTK_USE_VIDEO_FOR_WINDOWS   for video capture
        # TODO try VTK_USE_VIDEO_FOR_WINDOWS_CAPTURE   for video capture
        # TODO try VTK_USE_MICROSOFT_MEDIA_FOUNDATION   for video capture (MP4)

        # groups can be:  DEFAULT   DONT_WANT   WANT   YES   NO
        # Note that YES is like WANT, but will show errors if can't make everything
        # NO is also more forceful than DONT_WANT
        # Default is DONT_WANT, let it auto-enable when required
        tc.variables["VTK_GROUP_ENABLE_Imaging"]    = "DONT_WANT"   # TODO add option
        tc.variables["VTK_GROUP_ENABLE_MPI"]        = "DONT_WANT"   # TODO add option
        tc.variables["VTK_GROUP_ENABLE_Rendering"]  = "YES" if self.options.rendering else "DONT_WANT"
        tc.variables["VTK_GROUP_ENABLE_StandAlone"] = "DONT_WANT"   # TODO add option
        tc.variables["VTK_GROUP_ENABLE_Views"]      = "DONT_WANT"   # TODO add option
        tc.variables["VTK_GROUP_ENABLE_Web"]        = "DONT_WANT"   # TODO add option

        # for QT, use the more specific MODULE options below
        tc.variables["VTK_GROUP_ENABLE_Qt"] = "DONT_WANT"

        ##### QT ######
        # QT has a few modules, we'll be specific
        tc.variables["VTK_QT_VERSION"] = self.options.qt_version
        tc.variables["VTK_MODULE_ENABLE_VTK_GUISupportQt"]      = "YES" if self.options.qt else "DONT_WANT"
        tc.variables["VTK_MODULE_ENABLE_VTK_GUISupportQtQuick"] = "YES" if self.options.qtquick else "DONT_WANT"
        tc.variables["VTK_MODULE_ENABLE_VTK_GUISupportQtSQL"]   = "YES" if self.options.qtsql else "DONT_WANT"

        ##### SMP parallelism ####  Sequential  STDThread  OpenMP  TBB
        # Note that STDThread seems to be available by default
        tc.variables["VTK_SMP_IMPLEMENTATION_TYPE"] = "Sequential"
        # Change change the mode during runtime, if you enable the backends like so:
        # tc.variables["VTK_SMP_ENABLE_<backend_name>"] = "ON"

        # TODO OLD STUFF that was here before, I don't use or know much about
        # if self.options.minimal:
        #     tc.variables["VTK_Group_StandAlone"] = "OFF"
        #     tc.variables["VTK_Group_Rendering"] = "OFF"
        # if self.options.ioxml:
        #     tc.variables["Module_vtkIOXML"] = "ON"
        # if self.options.mpi:
        #     tc.variables["VTK_Group_MPI"] = "ON"
        #     tc.variables["Module_vtkIOParallelXML"] = "ON"
        # if self.options.mpi_minimal:
        #     tc.variables["Module_vtkIOParallelXML"] = "ON"
        #     tc.variables["Module_vtkParallelMPI"] = "ON"
        #
        # if self.settings.build_type == "Debug" and self.settings.compiler == "Visual Studio":
        #     tc.variables["CMAKE_DEBUG_POSTFIX"] = "_d"

        #### Use the Internal VTK bundled libraries for these Third Party dependencies ...
        # Ask VTK to use their bundled versions for these:
        #
        # VTK uses a heavily-forked version they call 2.4.0.  Upstream libharu is currently unmaintained.
        tc.variables["VTK_MODULE_USE_EXTERNAL_VTK_libharu"] = "OFF"
        #
        # CCI does not have gl2ps yet.  Note that cern-root is also waiting for gl2ps.
        tc.variables["VTK_MODULE_USE_EXTERNAL_VTK_gl2ps"] = "OFF"
        #
        ###

        # Everything else should come from conan.
        # TODO check if anything is coming from the system CMake,
        # and change it to conan or add as a build_requirement for the system to install.
        # SOME of VTK's bundled Third Party libs are heavily forked and patched, so system versions may
        # not be appropriate to use externally (like libharu).
        tc.variables["VTK_USE_EXTERNAL"] = "ON"

        tc.generate()

    def build(self):
        apply_conandata_patches(self)
        cmake = CMake(self)
        cmake.configure()
        cmake.build()

    def package(self):
        cmake = CMake(self)
        cmake.install()

    def package_info(self):
###         self.cpp_info.libs = tools.collect_libs(self)
#        if self.settings.os == "Linux":
#            self.cpp_info.system_libs += ["dl","pthread"]

        # this is still required as vtk files include themselves as <vtkCommand.h>,
        # and it can't seem to find itself in the same dir when included with <>
###         self.cpp_info.includedirs = [ "include/vtk" ] # no short_version as we disabled versioning

#            "include/vtk-%s" % self.short_version,
#            # dont know why these were in here ... "include/vtk-%s/vtknetcdf/include" % self.short_version,
#            # dont know why these were in here ... "include/vtk-%s/vtknetcdfcpp" % self.short_version
#        ]

        # self.cpp_info.names["cmake_find_package"] = "VTK"
        # self.cpp_info.names["cmake_find_package_multi"] = "VTK"

        # needed?
        # self.cpp_info.set_property("cmake_target_name", "VTK::VTK")
        # self.cpp_info.set_property("cmake_module_target_name", "VTK::VTK")

        # TRY WITHOUT ... self.cpp_info.set_property("cmake_find_mode", "both")
        self.cpp_info.set_property("cmake_file_name", "VTK")

        # needed? self.cpp_info.set_property("cmake_module_file_name", "vtk")

        self.cpp_info.names["cmake_find_package"] = "VTK"
        self.cpp_info.names["cmake_find_package_multi"] = "VTK"

        components = [
                  "CommonColor"
                , "CommonComputationalGeometry"
                , "CommonCore"
                , "CommonDataModel"
                , "CommonExecutionModel"
                , "CommonMath"
                , "CommonMisc"
                , "CommonSystem"
                , "CommonTransforms"
                , "DICOMParser"
                , "FiltersCore"
                , "FiltersExtraction"
                , "FiltersGeneral"
                , "FiltersGeometry"
                , "FiltersHybrid"
                , "FiltersModeling"
                , "FiltersSources"
                , "FiltersStatistics"
                , "FiltersTexture"
                , "GUISupportQt"
                , "ImagingColor"
                , "ImagingCore"
                , "ImagingGeneral"
                , "ImagingHybrid"
                , "ImagingSources"
                , "InteractionStyle"
                , "InteractionWidgets"
                , "IOCore"
                , "IOImage"
                , "IOLegacy"
                , "IOXML"
                , "IOXMLParser"
                , "kissfft"
                , "loguru"
                , "metaio"
                , "ParallelCore"
                , "ParallelDIY"
                , "RenderingAnnotation"
                , "RenderingContext2D"
                , "RenderingCore"
                , "RenderingFreeType"
                , "RenderingOpenGL2"
                , "RenderingUI"
                , "RenderingVolume"
                , "sys"
                , "WrappingTools"
                ]

        # Note: I don't currently import the explicit dependency list for each component from VTK,
        # so every module will depend on "everything" external, and there are no internal dependencies.
        # Consumers just have to figure out what they have to link.

        # get keys as a list and make a list of target::target
        all_targets = [k + "::" + k for k in self._third_party().keys()]

        # all modules use the same include dir,
        # and note we aren't using include/vtk-9
        # no short_version in path, as we disabled versioning
        vtk_include_dirs   = [os.path.join("include", "vtk")]

        # all modules use the same cmake dir too
        vtk_cmake_dirs = [os.path.join("lib","cmake","vtk")]

        for comp in components:
            self.cpp_info.components[comp].set_property("cmake_target_name", "VTK::" + comp)
            self.cpp_info.components[comp].libs          = ["vtk" + comp]
            self.cpp_info.components[comp].includedirs   = vtk_include_dirs
            self.cpp_info.components[comp].builddirs     = vtk_cmake_dirs
            self.cpp_info.components[comp].build_modules = vtk_cmake_dirs
            self.cpp_info.components[comp].requires      = all_targets


from dataclasses import dataclass
from typing import List, Optional

import pandas as pd

from app.models import cypher_diagram_models
from app.utils import hex_to_rgb
from app.utils.meta_graph import (
    color_palette,
    line_color_dict,
    meta_node_def,
    meta_rel_explore_url,
)
from app.utils.visjs_config import diagram_option, node_alpha

HIGHLIGHT_COLOR = color_palette["red"]["600"]

@dataclass
class DiagramNode:
    """
    - id: 1-indexed integer id
    - meta_node: Type of the node that should be referrable in the meta_graph
    - shape: visjs node shape
    """

    id: int
    meta_node: str
    label: Optional[str] = None
    sub_label: Optional[str] = None
    shape: str = "box"

    def __post_init__(self) -> None:
        if self.label is None:
            if self.sub_label is not None:
                self.label = f"{self.meta_node}\n{self.sub_label}"
            else:
                self.label = self.meta_node

    def generate_node(self):
        node = {
            "id": self.id,
            "label": self.label,
            "font": {"color": meta_node_def.loc[self.meta_node, "fg"]},
            "url": meta_node_def.loc[self.meta_node, "url"],
            "color": {
                "background": hex_to_rgb(
                    meta_node_def.loc[self.meta_node, "bg"], alpha=node_alpha
                ),
                "highlight": {
                    "background": meta_node_def.loc[self.meta_node, "bg"],
                    "border": HIGHLIGHT_COLOR,
                },
                "hover": {
                    "background": meta_node_def.loc[self.meta_node, "bg"],
                    "border": HIGHLIGHT_COLOR,
                },
            },
            "shape": "box",
        }
        return node

@dataclass
class DiagramEdge:
    """
    - from_id, to_id: 1-indexed integer id of the specific node
    - from_node: If specified, the edge will use the color of the node
    """

    from_id: int
    to_id: int
    meta_rel: Optional[str] = None
    label: Optional[str] = None
    sub_label: Optional[str] = None
    arrows: bool = True
    dashes: bool = False

    def __post_init__(self) -> None:
        self.url: Optional[str]
        if self.label is None:
            if self.meta_rel is not None and self.sub_label is not None:
                self.label = f"{self.meta_rel}\n{self.sub_label}"
            elif self.meta_rel is not None:
                self.label = self.meta_rel
        if self.meta_rel is not None:
            self.url = meta_rel_explore_url(self.meta_rel)
        else:
            self.url = None

    def generate_edge(self, node_df: pd.DataFrame):
        meta_node = node_df.at[self.from_id, "meta_node"]
        edge = {
            "from": self.from_id,
            "to": self.to_id,
            "arrows": {"to": self.arrows},
            "label": self.label,
            "url": self.url,
            "length": 100,
            "width": 2,
            "dashes": self.dashes,
            "color": {
                "color": meta_node_def.loc[meta_node, "bg"],
                "hover": line_color_dict["hover"],
            },
        }
        return edge

class CypherDiagram:
    def __init__(
        self,
        nodes: List[DiagramNode],
        edges: List[DiagramEdge],
        visjs_option=diagram_option,
    ):
        self.nodes = [node.generate_node() for node in nodes]
        nodes_df = pd.DataFrame(
            [{"id": node.id, "meta_node": node.meta_node} for node in nodes]
        ).set_index("id")
        self.edges = [edge.generate_edge(nodes_df) for edge in edges]
        self.option = visjs_option

    def generate_diagram(self) -> cypher_diagram_models.CypherDiagramData:
        self.diagram: cypher_diagram_models.CypherDiagramData = {
            "nodes": self.nodes,
            "edges": self.edges,
            "option": self.option,
        }
        return self.diagram


from json import dumps
from flask import Flask,Response, request
from neo4j import GraphDatabase, basic_auth
import string
import random

app = Flask(__name__)

url = "bolt://localhost:7687"
username = ""
password = ""

driver = GraphDatabase.driver(url, auth=basic_auth(username, password))

def get_db():
    return driver.session()

@app.route("/homepage")
def get_index():
    return app.send_static_file('index.html')

@app.route("/person")
def get_persondetail():
    return app.send_static_file('person.html')

@app.route("/graph")
def get_graph():
    print("graph called")
    db = get_db()
    results = db.read_transaction(lambda tx: list(tx.run("MATCH (a:Person) "
                                                         "RETURN a.name as name, a.surname as surname, "
                                                         "a.address as address, a.age as age, a.taxcode as taxcode"
                                                         )))
    nodes = []
    for record in results:
        nodes.append({"name": record["name"], "surname": record["surname"],
        "address": record["address"],"age": record["age"], "taxcode":record["taxcode"]})

    db.close()
    return Response(dumps({"nodes": nodes}),
                    mimetype="application/json")

@app.route("/search")
def get_search():
    try:
        p = request.args.get('person')
        p=p.split(" ")
        while p.count('') > 0: p.remove('')

    except Exception as e:
        print(e)
        return list()
    else:
        db = get_db()

        if len(p) == 1:
            results = db.read_transaction(lambda tx: list(tx.run("MATCH (a:Person) "
                                                             "WHERE a.name =~ $name "
                                                             "RETURN a.name as name, a.surname as surname, "
                                                         "a.address as address, a.age as age, a.taxcode as taxcode"
                                                         , name = p[0])))
        else:
            results = db.read_transaction(lambda tx: list(tx.run("MATCH (a:Person) "
                        "WHERE a.name =~ $name and a.surname =~ $surname "
                        "RETURN a.name as name, a.surname as surname, "
                                                         "a.address as address, a.age as age, a.taxcode as taxcode"
                                                         , name = p[0] , surname = p[1])))

        nodes = list()
        for record in results:
            nodes.append({"name": record["name"], "surname": record["surname"],
                "address": record["address"],"age": record["age"], "taxcode":record["taxcode"]})

        db.close()
        return Response(dumps({"nodes": nodes}),
                    mimetype="application/json")

@app.route("/covid-test")
def get_covid_test():
    print("covid test called")
    try:
        p = request.args.get('taxcode')
    except Exception as e:
        print(e)
        return list()
    db = get_db()
    results = db.read_transaction(lambda tx: list(tx.run(
        '''
        match(n:Person{taxcode: $taxcode})-[r]->(c:CovidTest) return c.id as id, c.type as type, c.date as date,
        c.result as result
        ''', taxcode = p
        )))
    nodes = []
    for record in results:
        nodes.append({"id": record["id"], "date": str(record["date"]),
        "type": record["type"],"result": record["result"]})
    db.close()
    return Response(dumps({"nodes": nodes}),
                    mimetype="application/json")

@app.route("/vaccine")
def get_vaccine():
    print("vaccine called")
    try:
        p = request.args.get('taxcode')
    except Exception as e:
        print(e)
    db = get_db()
    results = db.read_transaction(lambda tx: list(tx.run(
        '''
        match(n:Person{taxcode: $taxcode})-[r]->(v:VaccineCertificate) return v.id as id, v.type as type,
        v.first_dose_date as first_dose_date, v.second_dose_date as second_dose_date
        ''', taxcode = p
        )))
    nodes = []

    for i,record in enumerate(results):

        nodes.append({"id": record["id"],
        "type": record["type"],"first_dose_date": str(record["first_dose_date"])})
        try:
            nodes[i]["second_dose_date"] = str(record["second_dose_date"])
        except:
            print("No second dose")

    db.close()
    return Response(dumps({"nodes": nodes}),
                    mimetype="application/json")

@app.route("/create-covid-test")
def create_covid_test():
    try:

        d = {
            "type":request.args.get('type'),
            "date":request.args.get('date'),
            "result":request.args.get('result'),
            "taxcode": request.args.get("taxcode")
        }
    except Exception as e:
        print(e)
    else:
        db = get_db()

        res = db.read_transaction(lambda tx: list(tx.run(
            '''
                match(n:CovidTest) return n.id as id order by n.id DESC limit 1
            ''', )))
        highest_id = int(res[0]["id"])

        db.write_transaction(lambda tx: list(tx.run(
            '''
                MATCH(n:Person{taxcode:$taxcode}) CREATE (n)-[r:TESTED]->(a:CovidTest {id:$id, type:$type, result:$result, date:date($date)})
            ''',
            id=highest_id+1, type = d["type"], result=d["result"], date = d["date"],taxcode = d["taxcode"])))
        db.close()
        return {}

@app.route("/create-vaccine")
def create_vaccine():
    try:

        d = {
            "first_date":request.args.get('first_date'),
            "type":request.args.get('type'),
            "taxcode": request.args.get("taxcode")
        }
        print(d)
    except Exception as e:
        print(e)
    else:
        db = get_db()
        res = db.read_transaction(lambda tx: list(tx.run(
            '''
                match(n:VaccineCertificate) return n.id as id order by n.id DESC limit 1
            ''', )))
        highest_id = int(res[0]["id"])

        if d["type"] != "Johnson&Johnson":
            db.write_transaction(lambda tx: list(tx.run(
                '''
                    MATCH(n:Person{taxcode:$taxcode}) CREATE (n)-[r:VACCINATED]->
                    (a:VaccineCertificate {id:$id, type:$type, first_dose_date:date($first_date), second_dose_date:date($second_date)})
                ''',
                id=highest_id+1, type = d["type"], first_date=d["first_date"], second_date = request.args.get('second_date'),taxcode = d["taxcode"])))
        else:

            db.write_transaction(lambda tx: list(tx.run(
                '''
                    MATCH(n:Person{taxcode:$taxcode}) CREATE (n)-[r:VACCINATED]->
                    (a:VaccineCertificate {id:$id, type:$type, first_dose_date:date($first_date)})
                ''',
                id=highest_id+1, type = d["type"], first_date=d["first_date"],taxcode = d["taxcode"])))

        db.close()
        return {}

@app.route("/contacts")
def get_contacts():
    try:

        d = {
            "taxcode": request.args.get("taxcode")
        }

    except Exception as e:
        print(e)
    else:
        db = get_db()

        result = db.write_transaction(lambda tx: list(tx.run(
            '''
                MATCH(n:Person{taxcode:$taxcode})-[r]-(p:Person)
                return type(r) as rel_type,properties(r) as rel_prop,p
            ''',
            taxcode = d["taxcode"])))

        relationships = []
        for record in result:
            rel = {}
            rel["person_name"] = record["p"]["name"]
            rel["person_surname"] = record["p"]["surname"]
            rel["taxcode"] = record["p"]["taxcode"]

            type = record["rel_type"]
            rel["rel_type"] = type

            if type=="EVENT_CONTACT":
                rel["place"] = record["rel_prop"]["place"]
                rel["date"] = str(record["rel_prop"]["date"])
            elif type=="APP_CONTACT":
                rel["date"] = str(record["rel_prop"]["date"])
            relationships.append(rel)

        db.close()
        return Response(dumps({"relationships": relationships}),
                    mimetype="application/json")

@app.route("/create")
def create_person():
    try:
        d = {
            "name":request.args.get('name'),
            "surname":request.args.get('surname'),
            "age":request.args.get('age'),
            "address":request.args.get('address'),
        }
    except Exception as e:
        print(e)
        return list()
    else:
        db = get_db()

        db.write_transaction(lambda tx: list(tx.run(
            '''
                CREATE (a:Person {taxcode:$taxcode, name:$name, surname:$surname, age:$age, address:$address})
            ''', taxcode = ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)),
                    name = d["name"], surname = d["surname"], age = d["age"],address = d["address"])))

        db.close()
        return {}

@app.route("/create-relationship")
def create_relationship():
    try:

        d = {
            "first_taxcode":request.args.get('first_taxcode'),
            "second_taxcode":request.args.get('second_taxcode'),
            "type":request.args.get('type')
        }
    except Exception as e:
        print(e)
        return list()
    else:
        db = get_db()

        if d["type"] == "family":
            db.write_transaction(lambda tx: tx.run(
                '''
                    MATCH
                        (firstPerson:Person {taxcode:$first_taxcode}),
                        (secondPerson:Person {taxcode:$second_taxcode})
                    CREATE
                        (firstPerson)-[r:FAMILY_CONTACT]->(secondPerson)
                ''',first_taxcode = d["first_taxcode"],second_taxcode = d["second_taxcode"] ))

        elif d["type"] == "app":
            db.write_transaction(lambda tx: tx.run(
                '''
                    MATCH
                        (firstPerson:Person {taxcode:$first_taxcode}),
                        (secondPerson:Person {taxcode:$second_taxcode})
                    CREATE
                        (firstPerson)-[r:APP_CONTACT {date:date($date)}]->(secondPerson)
                ''', first_taxcode = d["first_taxcode"],second_taxcode = d["second_taxcode"],
                    date = request.args.get('date')))
        elif d["type"] == "event":
            print("event")
            db.write_transaction(lambda tx: tx.run(
                '''
                    MATCH
                        (firstPerson:Person {taxcode:$first_taxcode}),
                        (secondPerson:Person {taxcode:$second_taxcode})
                    CREATE
                        (firstPerson)-[r:EVENT_CONTACT {date:date($date), place:$place}]->(secondPerson)
                ''', first_taxcode = d["first_taxcode"],second_taxcode = d["second_taxcode"],
                    date = request.args.get('date'), place = request.args.get('place') ))
        db.close()
        return {}

@app.route('/delete-person')
def delete_person():
    try:
        taxcode = request.args.get('taxcode')
    except Exception as e:
        print(e)
        return list()

    db = get_db()
    db.write_transaction(lambda tx: tx.run(
            '''
                MATCH(personToDelete:Person {taxcode:$taxcode}) DETACH DELETE personToDelete
            ''', taxcode = taxcode))
    db.close()
    return {}

@app.route('/delete-covidtest')
def delete_covidtest():
    try:
        id = request.args.get('id')
        print(id)
    except Exception as e:
        print(e)
        return list()

    db = get_db()
    db.write_transaction(lambda tx: tx.run(
            '''
                MATCH(ct:CovidTest {id:$id}) DETACH DELETE ct
            ''', id = int(id)))
    db.close()
    return {}

@app.route('/delete-vaccine')
def delete_vaccine():
    try:
        id = request.args.get('id')
        print(id)
    except Exception as e:
        print(e)
        return list()

    db = get_db()
    db.write_transaction(lambda tx: tx.run(
            '''
                MATCH(vc:VaccineCertificate {id:$id}) DETACH DELETE vc
            ''', id = int(id)))
    db.close()
    return {}

@app.route('/delete-contact')
def delete_relationship():
    try:
        taxcode1 = request.args.get('taxcode1')
        taxcode2 = request.args.get('taxcode2')
        print(id)
    except Exception as e:
        print(e)
        return list()

    db = get_db()
    db.write_transaction(lambda tx: tx.run(
            '''
                MATCH(n:Person {taxcode:$taxcode1})-[r]-(p:Person {taxcode:$taxcode2}) delete r
            ''', taxcode1 = taxcode1, taxcode2 = taxcode2))
    db.close()
    return {}

if __name__ == '__main__':
    app.run()


import discord
from discord.ext import commands
import traceback
from configparser import ConfigParser
from utils.utils import con

parser = ConfigParser()
parser.read('config.ini')
try:
    logchannel = int(parser.get('CONFIG', 'log_channel'))
except Exception as e:
    print(f"Couldn't define logchannel: {e}")
    logchannel = None

class CMDError(commands.Cog):

    def __init__(self, bot):
        self.bot = bot

    @commands.Cog.listener()
    async def on_command_error(self, ctx, error):
        if isinstance(error, commands.CommandNotFound):
            return
        elif isinstance(error, discord.Forbidden):
            embed = discord.Embed(title="Error", description="""I do not have enough permissions to execute this command. I require the following permissions:\nRead Messages, Send Messages, Embed Links, Add Reactions""", color=0xff0000)
            await ctx.send(embed=embed)
            return
        elif isinstance(error, commands.MissingRequiredArgument):
            embed = discord.Embed(title="Error", description="""Missing Required Argument.""", color=0xff0000)
            await ctx.send(embed=embed)
            return
        elif isinstance(error, commands.NotOwner):
            embed = discord.Embed(title="Error", description="""This command is either restricted, or is temporarily disabled for testing purposes.""", color=0xff0000)
            await ctx.send(embed=embed)
            return
        elif isinstance(error, commands.CommandOnCooldown):
            embed = discord.Embed(title="Cooldown", description="""That command is on cooldown, you may use it again once all data has been retrieved.""", color=0xff0000)
            await ctx.send(embed=embed)
            return
        else:
            error = getattr(error, "original", error)
            embed = discord.Embed(title="Error", description="""An unknown error occurred. This error has been reported.
            ```\n""" + str(error) + '\n```', color=0xff0000)
            await ctx.send(embed=embed)
            try:
                raise error
            except:
                tb = traceback.format_exc()
            con.log(error)
            with open('utils\\logs\\error.log', 'a') as logfile:
                logfile.write(tb)
            if logchannel is not None:
                channel = self.bot.get_channel(logchannel)
                if channel is not None:
                    embed = discord.Embed(title=f"Exception in '{ctx.command}'", description=f"```\n{tb}\n```", color=0xff0000)
                    try:
                        await channel.send(embed=embed)
                    except discord.Forbidden:
                        raise ValueError("The bot does not have permissions to send messages in the logchannel specified in config.ini.")
                else:
                    raise ValueError("The logchannel specified in config.ini is not visible to the bot, or does not exist.")
            return

def setup(bot):
    bot.add_cog(CMDError(bot))


from __future__ import absolute_import
from __future__ import print_function
import os,sys,codecs,numpy as np,random,imp,time,random
#try:
    #import ujson as json
import simplejson as json
#except ImportError:
#	import json
from datetime import datetime as dt
import unicodecsv as csv
from collections import defaultdict,Counter
from .config import CONFIG
import six
from six.moves import range
from tqdm import tqdm
import random,shutil
import inspect
DEFAULT_PATH_KEY='_path'
DEFAULT_EXT = 'txt'

PATH_KEY=CONFIG.get('PATH_KEY','')
PATH_EXT=CONFIG.get('PATH_EXT','').replace('.','')

if not PATH_KEY: PATH_KEY=DEFAULT_PATH_KEY
if not PATH_EXT: PATH_EXT=DEFAULT_PATH_EXT

TXT_MAXCOLS=25000
from smart_open import open
from tqdm import tqdm

def slingshot_single_shot(stone,path):
    return stone(path)

def get_paths_already_finished_from_cache(cache_path):
    for path,result in stream_results(cache_path,flatten=False,progress=False):
        if result:
            if type(path)==list: path=tuple(path)
            if type(path)==dict: path=tuple(sorted(list(path.items())))
            yield path

def now(now=None):
    import datetime as dt
    if not now:
        now=dt.datetime.now()
    elif type(now) in [int,float,str]:
        now=dt.datetime.fromtimestamp(now)

    return '{0}-{1}-{2} {3}:{4}:{5}'.format(now.year,str(now.month).zfill(2),str(now.day).zfill(2),str(now.hour).zfill(2),str(now.minute).zfill(2),str(now.second).zfill(2))

data_slingshot='.data_slingshot/'

def shoot(func, objects, path_src=None, savedir=None, overwrite=False, **kwargs):
    # init savedir
    funcname = func if type(func)==str else func.__name__
    if type(func)==str and (not path_src or not os.path.exists(path_src)):
        print('`func` must be real function object, or if a string, `path_src` must be specified.')
        return

    # init
    if not savedir: savedir=f'_data_/{funcname}'
    # if not savedir: savedir=f'_data_/{funcname}/{now().split()[0]}'
    if os.path.exists(savedir) and overwrite: shutil.rmtree(savedir)
    if not os.path.exists(savedir): os.makedirs(savedir)
    if not 'resume' in kwargs: kwargs['resume']=True
    if not 'nosave' in kwargs: kwargs['nosave']=True
    path_code=f'{savedir}/func.py'
    path_objs=f'{savedir}/input.jsonl'

    # get source of function
    if type(func)!=str:
        func_txt=inspect.getsource(func)
        with open(path_code,'w') as of:
            of.write(func_txt)
    else:
        shutil.copyfile(path_src,path_code)

    # save objects
    with open(path_objs,'w') as of:
        for obj in tqdm(objects,total=len(objects),desc='[Slingshot] Preparing input'):
            of.write(json.dumps(obj)+'\n')

    # call command
    cmd=f'slingshot -code {path_code} -func {funcname} -pathlist {path_objs} -savedir {savedir} -resume'
    for k,v in kwargs.items():
        cmd+=f' -{k} {v}' if v is not True else f' -{k}'
    return cmd

def gather(savedir):
    return list(stream_results(os.path.join(savedir,'cache')))

def slingshot(path_sling=None,stone_name=None,stone_args=None,paths=None,llp_corpus=None,limit=None,path_source=None,stone=None,path_key=PATH_KEY,path_ext=None,path_prefix='',path_suffix='',cache_results=True,cache_path=None,save_results=True,results_dir=None,shuffle_paths=True,do_stream_results=True,save_txt=True,txt_maxcols=TXT_MAXCOLS,sling_args=[],sling_kwargs={},num_runs=1,oneshot=False,llp_pass_text=False,llp_method='',progress_bar=True,savecsv='',resume=False,overwrite=False,parallel=1,llp_pass_path='path'):
    """
    Main function
    """
    progress_bar=True# if not sbatch else False

    # Load code-sling and stone-function
    if not stone and not llp_method:
        stone=load_stone_in_sling(path_sling,stone_name)
        if not stone:
            return

    # One shot?
    if oneshot:
        return stone(path_source,*sling_args,**sling_kwargs)

    # Load paths
    all_paths = None
    if llp_corpus and llp_corpus!='None':
        try:
            import llp
            #print(llp,dir(llp))
            corpus = llp.load_corpus(llp_corpus)
            #print(corpus.path_metadata)
            #print(llp_corpus, corpus)
            #print('llp_pass_path',llp_pass_path)
            #print('llp_pass_text',llp_pass_text)
            #print('llp_method',llp_method)
            #print(corpus.texts())
            all_paths = [(text.addr if (llp_pass_text or llp_method) else getattr(text,llp_pass_path)) for text in corpus.texts()][:limit]
            #print(all_paths[:10])
            if shuffle_paths:
                import random
                random.shuffle(all_paths)
        except ImportError as e:
            print('!!',e)
            pass

    # Get paths!
    if not all_paths:
        all_paths = load_paths(path_source,path_ext,limit,shuffle_paths,path_key,path_prefix,path_suffix) if not paths else paths

    # Break if these weren't returned
    if not all_paths: return

    # Report back
    #print('>> [Slingshot] # of paths:',len(all_paths))

    # Cache stuff
    if not results_dir: results_dir=f'data_slingshot/{stone_name}'
    if not cache_path: cache_path=os.path.join(results_dir,'cache')
    if cache_results:
        if overwrite and os.path.exists(cache_path): delete_files_from_folder(cache_path)
        if overwrite and os.path.exists(results_dir): delete_files_from_folder(results_dir)
        if not os.path.exists(cache_path):
            try:
                os.makedirs(cache_path)
            except FileExistsError:
                pass
        if resume:
            paths_done = set(list(get_paths_already_finished_from_cache(cache_path)))
            print('\n>> [Slingshot] already finished %s of %s' % (len(paths_done),len(all_paths)))
            try:
                all_paths=list(set(all_paths)-paths_done)
            except TypeError:
                pass

        import random
        if shuffle_paths: random.shuffle(all_paths)

    ## de-tuple?
    all_paths = [
        dict(p) if (p and type(p)==tuple and set(len(x) for x in p)=={2}) else p
        for p in all_paths
    ]

    ## RUNS?
    # Multiply paths by runs
    all_paths = [(path,run+1) for path in all_paths for run in range(num_runs)]

    t1 = dt.now()
    if parallel < 2:
        paths=all_paths
        results=[]
        looper=all_paths if not progress_bar else tqdm(all_paths,file=sys.stdout,desc='Slingshot',position=0,ncols=100)
        if cache_results and cache_path:
            cache_fn = 'results.jsonl'
            cache_fnfn = os.path.join(cache_path,cache_fn)
            cache_writer = open(cache_fnfn, mode='a+',encoding='utf-8')
        rank = 0

    else:
        # Start MPI
        from mpi4py import MPI

        comm = MPI.COMM_WORLD
        size = comm.Get_size()
        rank = comm.Get_rank()
        #if rank==0: print()
        #print('>> [Slingshot] initializing MPI with size %s and rank %s' % (size,rank))

        # Am I the seed process?
        if rank == 0:
            # Farm out paths to other processes
            segments = np.array_split(all_paths,size) if size>1 else [all_paths]
            print('>> [Slingshot] %s paths divided into %s segments' % (len(all_paths), len(segments)))

        # Or am I a process created by the seed?
        else:
            segments = None

        # Scatter the segments (if rank is 0?)
        segment = comm.scatter(segments, root=0)

        # Parse the segment.
        paths = segment

        # cache results?
        cache_writer=None
        if cache_results and cache_path:
            cache_fn = 'results.rank=%s.jsonl' % str(rank).zfill(4)
            cache_fnfn = os.path.join(cache_path,cache_fn)
            cache_writer = open(cache_fnfn, mode='a+',encoding='utf-8')

        #progress_bar=True
        desc=f'>> [Slingshot] Compute worker {rank}'
        looper=paths if not progress_bar else tqdm(paths,file=sys.stdout,desc=desc,position=rank)

    # let's go! loop over the paths
    results=[]
    num_paths=len(paths)
    pronoun='their'
    zlen=len(str(num_paths))
    zlen_rank=3 #len(str(size))

    for i,(path,run) in enumerate(looper):
        #################################################
        # THIS IS WHERE THE STONE FITS INTO THE SLINGSHOT

        sling_kwargs2=dict(list(sling_kwargs.items()))
        #sling_kwargs2['results_dir']=results_dir
        if num_runs>1: sling_kwargs2['run']=run

        path_store = None

        if llp_pass_text and not llp_method:
            idx = path
            text = corpus.text(path)
            path_store = text.addr
            path = text.path_txt

        if llp_method:
            idx = path
            try:
                text = corpus.text(path)
                path = text.addr
                stone = getattr(text,llp_method)
            except AttributeError as e:
                print("!!",e)
                return
            result = stone(*sling_args,**sling_kwargs2)

        else:
            try:
                result=stone(path,*sling_args,**sling_kwargs2)
            except TypeError:
                result=stone(path,*sling_args,**sling_kwargs)

        if result is not None:
            path_store = path if not path_store else path_store
            path_result=(path_store,result)
            if not do_stream_results: results+=[path_result]
            #if cache_writer:
            if cache_writer:
                #cache_writer.write(path_result) # when using jsonlines
                #try:
                #jsonl=json.dumps(path_result)
                jsonl=json.dumps(path_result,ignore_nan=True)
                cache_writer.write(jsonl + '\n')
                #except:
                #	print "!! could not write to results file!"
        #################################################
        #print('>> Clone #%s slings %s at #%s of %s %s enemies!' % (str(rank).zfill(zlen_rank),stone_name,str(i+1).zfill(zlen),pronoun,num_paths))
        #print('>> Clone #%s slings %s at Target #%s (of %s)' % (str(rank).zfill(zlen_rank),stone_name,str(i+1).zfill(zlen),num_paths))
        #if not progress_bar: print('>> Clone #%s -- slings --> Target #%s / %s' % (str(rank).zfill(zlen_rank),str(i+1).zfill(zlen),num_paths))
    if cache_writer: cache_writer.close()

    # Gather the results
    if parallel>1: RESULTS = comm.gather(results, root=0)

    # If I am the seed process again
    if rank == 0:
        t3 = dt.now()
        print('>> [Slingshot] Finished parsing in %s seconds.' % (t3-t1).total_seconds())

        # Save results...
        if save_results and results_dir:
            # Make dir...
            if not os.path.exists(results_dir): os.makedirs(results_dir)

            # Copy pathlist
            results_fnfn_pathlist = os.path.join(results_dir,'pathlist.txt')
            results_fnfn_metadata = os.path.join(results_dir,'metadata.txt')
            save_results_pathlist(results_fnfn_pathlist,results_fnfn_metadata,all_paths,path_source)

            # Stream-save TSV
            results_fnfn_txt = os.path.join(results_dir,'results.txt') if not savecsv else savecsv
            if save_txt: save_results_txt(results_fnfn_txt,cache_path,txt_maxcols)

        # Exit
        t4 = dt.now()
        print('>> [Slingshot] Finished everything in %s seconds!' % (t4-t1).total_seconds())

def save_results_pathlist(results_fnfn_pathlist,results_fnfn_metadata,paths,path_source):
    with open(results_fnfn_pathlist,'w') as of:
        for (path,run) in paths:
            try:
                of.write(str(path)+'\n')
            except TypeError:
                of.write(str(path.addr) + '\n')

    path_pathlists = CONFIG.get('PATH_PATHLISTS','')
    pathlist_path_source = os.path.join(path_pathlists,path_source if path_source else '')

    #print(path_source,'!?!?!?')
    if not path_source:
        pass
    elif os.path.exists(path_source) and is_csv(path_source):
        from shutil import copyfile
        copyfile(path_source, results_fnfn_metadata)
    elif os.path.exists(pathlist_path_source) and is_csv(path_source):
        from shutil import copyfile
        copyfile(path_source, results_fnfn_metadata)

def load_stone_in_sling(path_sling,stone_name, exts=['','.py','.ipynb']):
    if not path_sling or not stone_name:
        print('!! sling or stone not specified')
        return
    elif not os.path.exists(path_sling):
        new_path=None
        for ext in exts:
            abs_path_sling_ext=os.path.join(CONFIG['PATH_SLINGS'],path_sling+ext)
            #print(abs_path_sling_ext)
            if os.path.exists(abs_path_sling_ext):
                new_path=abs_path_sling_ext
                break
        if not new_path:
            print("!!",path_sling,"does not exist")
            return
        path_sling=new_path

    if path_sling.endswith('.py'):
        try:
            import importlib.util
            spec = importlib.util.spec_from_file_location("sling", path_sling)
            sling = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(sling)
        except ImportError:
            import imp
            sling = imp.load_source('sling', path_sling)

        stone = getattr(sling,stone_name)
        return stone

    elif path_sling.endswith('.ipynb'):
        import nbimporter
        nbimporter.options['only_defs'] = CONFIG.get('NBIMPORTER_ONLY_DEFS',False)

        ppath,pfn = os.path.split(path_sling)
        pname,pext = os.path.splitext(pfn)

        NBL = nbimporter.NotebookLoader(path=[ppath])
        sling = NBL.load_module(pname)
        stone = getattr(sling,stone_name)
        return stone

    elif path_sling.endswith('.R'):
        from rpy2.robjects import r as R
        # load all source
        with open(path_sling) as f:
            code=f.read()

            R('library(RJSONIO)')
            rfunc = R(code)
            #print('done!')
            stone = lambda _path: rconvert(rfunc(_path))
            return stone

def get_all_paths_from_folder(rootdir,ext='.txt'):
    for root, subdirs, files in os.walk(rootdir,followlinks=True):
        for fn in files:
            if fn.endswith(ext):
                yield os.path.join(root,fn)

def get_paths_from_csv(_fnfn,path_key=PATH_KEY,path_ext=PATH_EXT,path_prefix='',path_suffix='',sep='\t'):
    paths=[]
    #with codecs.open(_fnfn,encoding='utf-8') as pf:
    if not path_key: path_key=DEFAULT_PATH_KEY
    with open(_fnfn) as pf:
        reader=csv.DictReader(pf,delimiter=sep)
        for dx in reader:
            path=dx.get(path_key,'')
            if not path: continue
            if path_prefix: path=os.path.join(path_prefix,path)
            if path_suffix: path=path+path_suffix
            if path: paths+=[path]
    return paths

def is_csv(_fnfn,sep='\t'):
    if not os.path.exists(_fnfn): return False
    if os.path.isdir(_fnfn): return False
    with open(_fnfn) as pf:
        first_line=pf.readline()
        return sep in first_line

def get_paths_from_pathlist(_fnfn,sep='\t',path_key=PATH_KEY,path_prefix='',path_suffix=''):
    _fnfn_is_csv=is_csv(_fnfn,sep=sep)
    if _fnfn_is_csv:
        return get_paths_from_csv(_fnfn,path_key=path_key,sep=sep,path_prefix=path_prefix,path_suffix=path_suffix)
    elif _fnfn.endswith('jsonl'):
        import json
        jsons=[]
        with open(_fnfn) as f:
            for ln in f:
                try:
                    d=json.loads(ln.strip())
                    jsons.append(d)
                except (ValueError,json.JSONDecodeError) as e:
                    pass
        return jsons
    else:
        with open(_fnfn) as pf:
            paths=[line.strip() for line in pf]
            paths=[x for x in paths if x]
            if path_prefix: paths=[os.path.join(path_prefix,x) for x in paths]
            if path_suffix: paths=[path+path_suffix for x in paths]
            return paths

def load_paths(path_source,path_ext,limit,shuffle_paths,path_key=PATH_KEY,path_prefix='',path_suffix=''):
    paths=None
    if path_source:
        if os.path.isdir(path_source):
            paths=list(get_all_paths_from_folder(path_source,path_ext)) if path_source else None
        elif os.path.exists(path_source):
            paths=get_paths_from_pathlist(path_source,path_key=path_key,path_prefix=path_prefix,path_suffix=path_suffix)
        else:
            path_pathlists = CONFIG.get('PATH_PATHLISTS','')
            pathlist_path_source = os.path.join(path_pathlists,path_source)
            if os.path.exists(pathlist_path_source):
                paths=get_paths(pathlist_path_source)
    #paths=[p for p in paths if os.path.exists(p)]
    if not paths:
        print('!! no paths given or found at %s' % path_source if path_source else '')
        return
    #paths=sorted(list(set(paths)))
    paths=list(paths)
    paths=paths[:limit]
    if shuffle_paths:
        random.shuffle(paths)
    paths=[tuple(x) if type(x)==list else x for x in paths]
    paths=[tuple(sorted(list(x.items()))) if x==dict else x for x in paths]
    return paths

def save_results_txt(results_fnfn_txt,path_cache,txt_maxcols=TXT_MAXCOLS,sep='\t'):
    now=time.time()

    # First find KEYS
    KEYS=set()
    if txt_maxcols: Count=Counter()

    for path,result in stream_results(path_cache,flatten=True):
        if hasattr(result,'keys'):
            if txt_maxcols:
                Count.update(list(result.keys()))
            else:
                KEYS=set(result.keys())
    if txt_maxcols: KEYS={x for x,y in Count.most_common(txt_maxcols)}

    then,now=now,time.time()
    #print('>> save_txt: found %s keys in %ss' % (len(KEYS),int(now-then)))
    #print(list(KEYS)[:100],'...')

    if KEYS:
        # Then loop again to write
        header=['_path']+sorted([six.text_type(x) for x in KEYS])
        #with codecs.open(results_fnfn_txt,'w',encoding='utf-8') as results_f_txt:#, jsonlines.open(results_fnfn_json) as reader:
        with open(results_fnfn_txt,'w') as results_f_txt:
            results_f_txt.write(sep.join(header) + '\n')
            for path,result in stream_results(path_cache,flatten=True):
                result['_path']=path
                orow=[str(result.get(h,'')).replace(sep,' ').replace('\r\n',' ').replace('\r',' ').replace('\n',' ') for h in header]
                results_f_txt.write(sep.join(orow) + '\n')
            #print('>> saved:',results_fnfn_txt)
            then,now=now,time.time()
            #print('>> save_txt: saved in %ss' % int(now-then))

def rconvert(robj):
    from rpy2.robjects import r as R
    from rpy2.robjects import pandas2ri
    pandas2ri.activate()
    rjson=R['toJSON'](robj, collapse=' ')
    rjson_str = pandas2ri.ri2py(rjson)[0]
    pyobj = json.loads(rjson_str)
    sys.stdout.flush()
    return pyobj

### Loading results

def stream_results(path_cache,ext='.jsonl',flatten=False,progress=True):
    if 'jsonl' in os.path.basename(path_cache).split('.'):
        for path,data in stream_jsonl(path_cache,flatten=flatten,progress=progress):
            #if '.ipynb' in path: continue
            yield (path,data)
    else:
        for fn in sorted(os.listdir(path_cache)):
            if (not fn.endswith(ext) and not fn.endswith(ext+'.gz')): continue
            fnfn=os.path.join(path_cache,fn)
            #print('>> streaming:',fnfn,'...')
            for path,data in stream_jsonl(fnfn,flatten=flatten,progress=progress):
                #if '.ipynb' in path: continue
                yield (path,data)

def stream_jsonl(fn,flatten=False,progress=True):
    #from xopen import xopen
    import ujson as json
    if progress: from tqdm import tqdm

    #with xopen(fn) as f:
    #print('>> [Slingshot] streaming:',fn,'...')
    if progress: num_lines = get_num_lines(fn)
    with open(fn) as f:
        looper = tqdm(f,total=num_lines,desc=f'[Slingshot] Reading {os.path.basename(fn)}') if progress else f
        for ln in looper:
            try:
                path,data=json.loads(ln)
            except ValueError:
                continue
            if not flatten or type(data)!=list:
                yield (path,data)
            else:
                for i,data_x in enumerate(data):
                    yield (path if not i else '',data_x)

def writegen(fnfn,generator,header=None,args=[],kwargs={}):
    if 'jsonl' in fnfn.split('.'): return writegen_jsonl(fnfn,generator,args=args,kwargs=kwargs)
    with open(fnfn,'w') as of:
        for i,dx in enumerate(generator()):
            if not header: header=sorted(dx.keys())
            if not i: of.write('\t'.join(header) + '\n')
            of.write('\t'.join([six.text_type(dx.get(h,'')) for h in header]) + '\n')
    print('>> saved:',fnfn)

def get_num_lines(filename):
    from smart_open import open

    def blocks(files, size=65536):
        while True:
            b = files.read(size)
            if not b: break
            yield b

    with open(filename, 'r', errors='ignore') as f:
        numlines=sum(bl.count("\n") for bl in blocks(f))

    return numlines

def iter_move(fn,force=False,prefix=''):
    if os.path.exists(fn):
        iter_fn=iter_filename(fn,force=force,prefix=prefix)
        iter_dir=os.path.dirname(iter_fn)
        if not os.path.exists(iter_dir): os.makedirs(iter_dir)
        shutil.move(fn,iter_fn)
        print(f'>> moved: {fn} --> {iter_fn}')

def iter_filename(fnfn,force=False,prefix='',inbetweener='_'):
    if os.path.exists(fnfn) or force:
        fndir,fn=os.path.split(fnfn)
        filename,ext = os.path.splitext(fn)
        fnum=2 if not force else 1
        maybe_fn=os.path.join(fndir, prefix + filename + ext)
        while os.path.exists(maybe_fn):
            fnum+=1
            maybe_fn=os.path.join(fndir, prefix + filename + inbetweener + str(fnum) + ext)
        fnfn = maybe_fn
    return fnfn

def delete_files_from_folder(path,toprint=True):
    if os.path.exists(path):
        for root, dirs, files in os.walk(path):
            for fn in files:
                fnfn=os.path.join(root, fn)
                os.unlink(fnfn)
                if toprint: print('>> deleted:',fnfn)


import torch
import torch.nn as nn

class NeuralNetwork(torch.nn.Module):
    def __init__(
            self, hidden_layers=[20, 20], dim_in=2,
            dim_out=1, activation=torch.tanh):
        '''Initialiser method for Neural Network

        Args:
        '''
        super().__init__()  # Call from superclass

        self.activation = activation
        layers = [dim_in] + hidden_layers + [dim_out]
        self.num_layers = len(layers)
        self.layers = nn.ModuleList([
            nn.Linear(layers[i], layers[i+1])
            for i in range(self.num_layers-1)])

    def forward(self, x, t):  # forward pass on NN with activation function
        # tanh acivation function
        u = torch.cat([x, t], dim=1)

        for i in range(self.num_layers-1):
            u = self.activation(self.layers[i](u))
        return u