''' Example script to run a full analysis on telescope data for BCGS lab course. The script calls
    typical steps of test beam data reconstruction. For simplicity fits are fitted with a straight line.

    Setup
    -----

    The telescope consists of 5 planes with 15cm clearance between the planes.
    The data was taken at ELSA with 2.5 GeV/c.

    The Mimosa26 has an active area of 21.2mm x 10.6mm and the pixel matrix
    consists of 1152 columns and 576 rows (18.4um x 18.4um pixel size).
    The total size of the chip is 21.5mm x 13.7mm x 0.036mm
    (radiation length 9.3660734)

    The matrix is divided into 4 areas. For each area the threshold can be set up
    individually. The quartes are from column 0-287, 288-575, 576-863 and 864-1151.
'''

import os
import inspect
import logging

from beam_telescope_analysis import hit_analysis
from beam_telescope_analysis import dut_alignment
from beam_telescope_analysis import track_analysis
from beam_telescope_analysis import result_analysis
from beam_telescope_analysis.tools import data_selection, analysis_utils
from beam_telescope_analysis.telescope.telescope import Telescope

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - [%(levelname)-8s] (%(threadName)-10s) %(message)s")

def run_analysis(hit_files):
    # Create output subfolder where all output data and plots are stored
    output_folder = os.path.join(os.path.split(hit_files[0])[0], 'output_eutelescope')
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    mask_files = [(os.path.splitext(hit_file)[0] + '_mask.h5') for hit_file in hit_files]
    cluster_files = [(os.path.splitext(hit_file)[0] + '_clustered.h5') for hit_file in hit_files]

    z_positions = [0.0, 150000.0, 300000.0, 450000.0, 600000.0]  # in um. To be adjusted.
    material_budget = [7e-4, 7e-4, 7e-4, 7e-4, 7e-4]  # 5 Mimosa26 planes
    initial_configuration = os.path.join(output_folder, 'telescope.yaml')
    telescope = Telescope()
    telescope.add_dut(dut_type="Mimosa26", dut_id=0, translation_x=0, translation_y=0, translation_z=z_positions[0], rotation_alpha=0, rotation_beta=0, rotation_gamma=0, material_budget=material_budget[0], name="Telescope 1")
    telescope.add_dut(dut_type="Mimosa26", dut_id=1, translation_x=0, translation_y=0, translation_z=z_positions[1], rotation_alpha=0, rotation_beta=0, rotation_gamma=0, material_budget=material_budget[1], name="Telescope 2")
    telescope.add_dut(dut_type="Mimosa26", dut_id=2, translation_x=0, translation_y=0, translation_z=z_positions[2], rotation_alpha=0, rotation_beta=0, rotation_gamma=0, material_budget=material_budget[2], name="Telescope 3")
    telescope.add_dut(dut_type="Mimosa26", dut_id=3, translation_x=0, translation_y=0, translation_z=z_positions[3], rotation_alpha=0, rotation_beta=0, rotation_gamma=0, material_budget=material_budget[3], name="Telescope 4")
    telescope.add_dut(dut_type="Mimosa26", dut_id=4, translation_x=0, translation_y=0, translation_z=z_positions[4], rotation_alpha=0, rotation_beta=0, rotation_gamma=0, material_budget=material_budget[4], name="Telescope 5")
    telescope.save_configuration(initial_configuration)
    prealigned_configuration = os.path.join(output_folder, 'telescope_prealigned.yaml')
    aligned_configuration = os.path.join(output_folder, 'telescope_aligned.yaml')

    # Generate noisy pixel mask for all DUTs
    thresholds = [2, 2, 2, 2, 2]
    # last plane has noisy cluster, use larger median filter to mask cluster
    pixel_mask_names = ["NoisyPixelMask", "NoisyPixelMask", "NoisyPixelMask", "NoisyPixelMask", "NoisyPixelMask"]
    mask_files = hit_analysis.mask(
        telescope_configuration=initial_configuration,
        input_hit_files=hit_files,
        pixel_mask_names=pixel_mask_names,
        thresholds=thresholds)

    # Cluster hits from all DUTs
    use_positions = [False, False, False, False, False]
    min_hit_charges = [0, 0, 0, 0, 0]
    max_hit_charges = [1, 1, 1, 1, 1]
    column_cluster_distances = [3, 3, 3, 3, 3]
    row_cluster_distances = [3, 3, 3, 3, 3]
    frame_cluster_distances = [0, 0, 0, 0, 0]
    cluster_files = hit_analysis.cluster(
        telescope_configuration=initial_configuration,
        select_duts=None,
        input_hit_files=hit_files,
        input_mask_files=[None if val else mask_files[i] for i, val in enumerate(use_positions)],
        use_positions=use_positions,
        min_hit_charges=min_hit_charges,
        max_hit_charges=max_hit_charges,
        column_cluster_distances=column_cluster_distances,
        row_cluster_distances=row_cluster_distances,
        frame_cluster_distances=frame_cluster_distances)

    # Correlate each DUT with the first DUT
    hit_analysis.correlate(
        telescope_configuration=initial_configuration,
        input_files=cluster_files,
        output_correlation_file=os.path.join(output_folder, 'Correlation.h5'),
        resolution=(100.0, 100.0),
        select_reference_duts=0)

    # Create pre-alignment, take first DUT as reference
    prealigned_configuration = dut_alignment.prealign(
        telescope_configuration=initial_configuration,
        input_correlation_file=os.path.join(output_folder, 'Correlation.h5'),
        reduce_background=True,
        select_reference_dut=0)

    # Merge all cluster tables into a single table
    hit_analysis.merge_cluster_data(
        telescope_configuration=initial_configuration,
        input_cluster_files=cluster_files,
        output_merged_file=os.path.join(output_folder, 'Merged.h5'))

    # Create alignment, take first and last DUT as reference ("select_telescope_duts" parameter)
    # The position (translation and rotation) of telescope DUTs are not changed
    aligned_configuration = dut_alignment.align(
        telescope_configuration=prealigned_configuration,
        input_merged_file=os.path.join(output_folder, 'Merged.h5'),
        select_duts=[[0, 1, 2, 3, 4]],  # align the telescope planes first
        select_telescope_duts=[0, 1, 2, 3, 4],  # telescope planes
        select_fit_duts=[[0, 1, 2, 3, 4]],
        select_hit_duts=[[0, 1, 2, 3, 4]],
        max_iterations=[5],
        max_events=(100000),
        track_chi2=15.0,
        quality_distances=[(18.4 * 2, 18.4 * 2), (18.4 * 2, 18.4 * 2), (18.4 * 2, 18.4 * 2), (18.4 * 2, 18.4 * 2), (18.4 * 2, 18.4 * 2)],
        isolation_distances=(1000.0, 1000.0),
        use_limits=True,
        plot=True)

    # Find tracks from the tracklets and create a track candidates table
    track_analysis.find_tracks(
        telescope_configuration=aligned_configuration,
        input_merged_file=os.path.join(output_folder, 'Merged.h5'),
        output_track_candidates_file=os.path.join(output_folder, 'TrackCandidates_aligned.h5'),
        align_to_beam=True)

    # Fit the track candidates, assign quality flags, and create a track table
    # Simple straight line fit
    track_analysis.fit_tracks(
        telescope_configuration=aligned_configuration,
        input_track_candidates_file=os.path.join(output_folder, 'TrackCandidates_aligned.h5'),
        output_tracks_file=os.path.join(output_folder, 'Tracks_aligned.h5'),
        select_duts=[0, 1, 2, 3, 4],
        select_fit_duts=[0, 1, 2, 3, 4],
        select_hit_duts=[0, 1, 2, 3, 4],
        exclude_dut_hit=True,
        quality_distances=[(18.4 * 2, 18.4 * 2), (18.4 * 2, 18.4 * 2), (18.4 * 2, 18.4 * 2), (18.4 * 2, 18.4 * 2), (18.4 * 2, 18.4 * 2)],
        isolation_distances=(1000.0, 1000.0),
        use_limits=False,
        plot=True)

    # Calculate the unconstrained residuals from all tracks
    result_analysis.calculate_residuals(
        telescope_configuration=aligned_configuration,
        input_tracks_file=os.path.join(output_folder, 'Tracks_aligned.h5'),
        output_residuals_file=os.path.join(output_folder, 'Residuals_aligned.h5'),
        select_duts=[0, 1, 2, 3, 4],
        nbins_per_pixel=20,
        use_limits=True)

    # Do additional track selection cuts on the tracks table
    data_selection.select_tracks(
        telescope_configuration=aligned_configuration,
        input_tracks_file=os.path.join(output_folder, 'Tracks_aligned.h5'),
        output_tracks_file=os.path.join(output_folder, 'Tracks_aligned_selected.h5'),
        select_duts=[0, 1, 2, 3, 4],
        select_hit_duts=[0, 1, 2, 3, 4],
        select_no_hit_duts=None,
        select_quality_duts=[[1, 2, 3, 4], [0, 2, 3, 4], [0, 1, 3, 4], [0, 1, 2, 4], [0, 1, 2, 3]],
        query='(track_chi2 < 15.0)')

    # Calculate the unconstrained residuals from final tracks (with chi^2 cut and quality selection)
    result_analysis.calculate_residuals(
        telescope_configuration=aligned_configuration,
        input_tracks_file=os.path.join(output_folder, 'Tracks_aligned_selected.h5'),
        output_residuals_file=os.path.join(output_folder, 'Residuals_aligned_selected.h5'),
        select_duts=[0, 1, 2, 3, 4],
        nbins_per_pixel=20,
        use_limits=True)

# Main entry point is needed for multiprocessing under Windows
if __name__ == '__main__':
    # Get the absolute path of example data
    tests_data_folder = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), 'data')
    # The location of the data files, one file per DUT
    hit_files = [analysis_utils.get_data(
        path='examples/TestBeamData_Mimosa26_DUT%d.h5' % i,
        output=os.path.join(tests_data_folder, 'TestBeamData_Mimosa26_DUT%d.h5' % i)) for i in range(5)]

    run_analysis(hit_files=hit_files)


import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import torch
import copy

class ImageDataset():
    def __init__(self, imgPath="./data/images.npy", labelPath="./data/labels.npy", boxPath="./data/boxes.npy", boxLabelPath="./data/boxLabels.npy", countsPath="./data/counts.csv", range=None):
        # self.device = torch.device("cpu")
        self.device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
        self.count = pd.read_csv(countsPath)
        self.range = len(self.count) if range is None else range
        self.images = preprocess(np.load(imgPath).astype('float32'))[0:range]
        self.labels = np.load(labelPath).astype('float32')[0:range]
        self.boxes = np.load(boxPath).astype('float32')[0:range]
        self.boxLabels = np.load(boxLabelPath).astype('float32')[0:range]
        self.prune()
        self.train_test_split()

    def prune(self):
        # prune data to only include images with more than 1 label
        self.index_map = []
        counts = self.count.sum(axis=1)
        new_images, new_labels, new_boxes, new_boxesLabels = [], [], [], []
        for i in range(len(self.images)):
            if counts[i] > 0:
                new_images.append(self.images[i])
                new_labels.append(self.labels[i])
                new_boxes.append(self.boxes[i])
                new_boxesLabels.append(self.boxLabels[i])
                self.index_map.append(i)
        self.images = np.array(new_images)
        self.labels = np.array(new_labels)
        self.boxes = np.array(new_boxes)
        self.boxLabels = np.array(new_boxesLabels)

    def train_test_split(self):
        indexes = list(range(len(self.images)))
        train_idx, val_idx = train_test_split(indexes, test_size=0.2, random_state=42)
        train_img_data, train_box_data, train_boxLabel_data, train_labels = [], [], [], []
        val_img_data, val_box_data, val_boxLabel_data, val_labels = [], [], [], []
        for i in range(len(train_idx)):
            train_img_data.append(self.images[train_idx[i]])
            train_box_data.append(self.boxes[train_idx[i]])
            train_boxLabel_data.append(self.boxLabels[train_idx[i]])
            train_labels.append(self.labels[train_idx[i]])
        for i in range(len(val_idx)):
            val_img_data.append(self.images[val_idx[i]])
            val_box_data.append(self.boxes[val_idx[i]])
            val_boxLabel_data.append(self.boxLabels[val_idx[i]])
            val_labels.append(self.labels[val_idx[i]])
        self.train_data = (train_img_data, train_box_data, train_boxLabel_data, train_labels)
        self.val_data = (val_img_data, val_box_data, val_boxLabel_data, val_labels)

    def get_batches(self, batch_size, training=True):
        data = self.train_data if training else self.val_data
        indexes = list(range(len(data[0])))
        perm_idx = np.random.permutation(indexes) if training else indexes
        for i in range(len(data[0]) // batch_size):
            batch_idx = perm_idx[i * batch_size : (i + 1) * batch_size]
            img_batch, target_batch = [], []
            for j in batch_idx:
                img = copy.deepcopy(data[0][j])
                img = torch.tensor(np.transpose(img, axes=[2, 0, 1])).to(self.device)
                img = img.to(self.device)
                img_batch.append(img)
                target_batch.append(self.make_target(data[1][j], data[2][j], data[3][j]))
            batch = (img_batch, target_batch)
            yield batch

    def make_target(self, boxes, boxLabels, label):
        data = {}
        img_boxes = []
        img_class_labels = []
        img_masks = []
        for i in range(len(boxes)):
            # while there is still boxes left
            if (boxLabels[i][0] != 0 and boxLabels[i][1] != 0):
                invalid_box = False
                for j in range(4):
                    if boxes[i][j] < 0 or boxes[i][j] >= 255:
                        invalid_box = True
                if invalid_box:
                    continue
                img_boxes.append(boxes[i])
                # classes are 0-indexed.
                img_class_labels.append(np.int64(boxLabels[i][1] - 1))
                img_mask = np.array(label[..., 0] == boxLabels[i][0], dtype=np.uint8)
                img_masks.append(img_mask)
            else:
                break
        data['boxes'] = torch.tensor(np.stack(img_boxes)).to(self.device)
        data['labels'] = torch.tensor(np.stack(img_class_labels)).to(self.device)
        data['masks'] = torch.tensor(np.stack(img_masks)).to(self.device)
        return data

def preprocess(images):
    # preprocess images
    images = images / 255
    images = np.transpose(images, axes=[3, 0, 1, 2])
    # normalize each channel
    for i in range(len(images)):
        mean = np.mean(images[i])
        std = np.std(images[i])
        images[i] = (images[i] - mean) / std
    images = np.transpose(images, axes=[1, 2, 3, 0])
    return images


import QuantLib as ql
import datetime as dt

def date_to_quantlib(date):
    if isinstance(date, dt.date):
        return ql.Date(date.day, date.month, date.year)
    elif isinstance(date, ql.Date):
        return date
    else:
        raise ValueError

def daterange(start_date, end_date):
    for n in range(int((end_date - start_date).days)):
        yield start_date + dt.timedelta(n)


# This file trained the neural network that we used for our Behavioral Cloning agent. It reads in the data
# saved from the BCNetDataGenerator file, modifies it as needed, then trains on all of it.

import os
import keras.backend
import tensorflow as tf
from tensorflow import keras
from keras import layers
from keras import backend as k
from keras.layers import Dense
import sys
import numpy
from sklearn.model_selection import train_test_split
from tqdm import tqdm

load_directory = "D:\\smashdataset\\parseddata\\"

# Loop over every file in the directory where we saved all our data.
for directory, subdirectory, data_files in os.walk(load_directory):
    # This will contain all of the gamestate data for the games in a directory.
    x_games = []
    # This will contain all of the controller data for the player that the NN is playing as.
    y_games = []
    # This line was only used once, hence it being commented out, but this is used to normalize stage
    # data because I forgot to do that in gathering the data.
    #enclosing_folder = ""

    # We train per stage directory rather than loading in all the files at once. It turns out that even with
    # memory mapped arrays, we didn't have enough memory to do everything at once.
    for file in tqdm(data_files):
        # Again, this is used for normalization.
        #enclosing_folder = os.path.basename(directory)

        # There were three crashes throughout the course of the data writing process.
        # That may mean incomplete data. Accidentally having some arrays a few entries
        # off from others would defeat the purpose of this exercise, hence all of this code.
        #
        # Split components of the filenames:
        file_breakdown = file.split("-")
        # Start with the gamestate data files:
        if file_breakdown[3] == "x.npy":
            # Ensure that the X file has a matching y0 and y1 file:
            if (
                    f"{file_breakdown[0]}-{file_breakdown[1]}-{file_breakdown[2]}-y0.npy" in data_files
                    and
                    f"{file_breakdown[0]}-{file_breakdown[1]}-{file_breakdown[2]}-y1.npy" in data_files
            ):
                # If p1 is a Fox player, load in the X data and the Y0 data. The data proved to be far more Fox heavy
                # than Jigglypuff heavy, so we chose to disregard any files that didn't have Fox in them.
                if file_breakdown[1] == "1":
                    path = os.path.join(directory, file)
                    # Note that we're loading these in memory mapped. This is because there is not enough memory to load the entire
                    # set in in most cases. We have >100GB of data.
                    x_games.append(numpy.load(path, mmap_mode='r+'))
                    y_games.append(numpy.load(f"{path[:-5]}y0.npy", mmap_mode='r+'))
                # If p1 is not a Fox player but p2 is, load in the X data and the Y1 data.
                elif file_breakdown[2] == "1":
                    path = os.path.join(directory, file)
                    x_games.append(numpy.load(path, mmap_mode='r+'))
                    y_games.append(numpy.load(f"{path[:-5]}y1.npy", mmap_mode='r+'))

    # This is the part that actually normalizes the stage data in each of these arrays. Once this was done once, it was
    # saved to disk, so we didn't have to do it all again. Note that we divide by 26 rather than 29 because the stage represented
    # by 29 had zero replays, so we divided by the next highest value.
    # for x in tqdm(x_games):
    #     new_x = int(enclosing_folder)/26
    #     x[:, 0] = new_x

    # If there's nothing in one of the subdirectories, go to the next one.
    if len(x_games) == 0:
        continue

    # Now that we've loaded in all of the game data and we know that input frames map to output frames, we don't care about which
    # games they came from. This code will flatten out the arrays such that they are treated as one continuous set of
    # input data/output data pairs.
    x_games = numpy.concatenate(x_games, axis=0)
    y_games = numpy.concatenate(y_games, axis=0)
    print("Concatenated.")

    # Convert to dtype=float32 to speed up training. These are stored as float64, and since things took awhile as-is I can only imagine how bad
    # it would've been had we not done this.
    x_games = x_games.astype(numpy.float32, copy=False)
    y_games = y_games.astype(numpy.float32, copy=False)
    print("Casted.")

    # Ensure that things were done right with the pruning up above. Want to make sure that every gamestate has a matching controller input.
    num_entries = tf.shape(x_games)[0]
    if num_entries != tf.shape(y_games)[0]:
        print("ERROR, Y entries not match X entries.")
        sys.exit(1)

    # Divide into training/testing sets using scikitlearn, 20% test and 80% train.
    x_train, x_test, y_train, y_test = train_test_split(x_games, y_games, test_size=0.2, random_state=20)

    # If this isn't the first loop, load in the model, otherwise we'll create it.
    if os.path.isdir(f"saved_symmetric_melee_model"):
        bc_model = keras.models.load_model(f"saved_symmetric_melee_model")
        print("Loaded model.")

    else:
        # Define model. The details of our various parameter changes and experiments are outlined in our writeup doc, but we tried several different
        # configurations here. Generally, though, this was patterned off of the model in an "Intro to TensorFlow" video tutorial series we found on
        # YouTube. The particular video that helped us out the most can be found here: https://youtu.be/pAhPiF3yiXI
        bc_model = keras.Sequential([
            # Input layer.
            keras.Input(shape=23),
            # This is one of our symmetric models, with the extra hidden layer.
            layers.Dense(115, activation='relu'),
            layers.Dense(115, activation='relu'),
            layers.Dense(115, activation='relu'),
            # We also tried making this layer a sigmoid activation instead of ReLU, thinking it might give better output results.
            layers.Dense(9, activation='relu'),

        ])

        bc_model.compile(
            # The loss function used in the example video didn't work with our data set, so we dug around and decided MSE was the best option.
            # Evidently it's the default for a reason and you really shouldn't mess with it unless you're sure you need to. Here's one source
            # that explained this to us: https://machinelearningmastery.com/how-to-choose-loss-functions-when-training-deep-learning-neural-networks/
            loss='mean_squared_error',
            # Again, just what was in the video. We would've experimented with more possible values/optimizers had we had the time.
            optimizer=keras.optimizers.Adam(learning_rate=0.001),
            # Our goal is to be as close as possible to human inputs at any given state, which is why we use accuracy.
            metrics=["accuracy"]
        )

    # Train the network. The batch size was settled on because we wanted frequent updates but we also wanted the process to not take far too long.
    # Ultimately, we probably could've afforded to increase this a bit, but likely to even worse results. We probably would've toyed with things like batch
    # size and number of epochs more had we had more time to spend training.
    bc_model.fit(x_train, y_train, batch_size=512, epochs = 50, verbose = 2, shuffle=True)
    # Show test results at the end of each training step.
    bc_model.evaluate(x_test, y_test, batch_size=512, verbose=2)
    # Save the network so that the next loop can pick up here.
    bc_model.save(f"saved_symmetric_melee_model/")

    # Delete arrays to make space for next loop.
    del(x_games)
    del(y_games)

    # Free VRAM
    keras.backend.clear_session()#


import os
import pathlib
import sys

input_file = sys.argv[1]

out_alleles = pathlib.Path('overall_alleles.tsv')
out_stats = pathlib.Path('overall_statistics.tsv')
with open(input_file, 'r') as f:
    for i in f.readlines():
        l = i.strip()
        p = pathlib.Path(l)
        for f in p.iterdir():
            if f.name == 'results_statistics.tsv':
                stat = open(f, 'r')
                if not out_stats.exists():
                    with open(out_stats, 'w') as out:
                        for l in stat.readlines():
                            out.write(l)
                        out.write('\n')
                else:
                    with open(out_stats, 'a') as out:
                        out.write(stat.readlines()[1])
                        out.write('\n')
            if f.name == 'results_alleles.tsv':
                allele = open(f, 'r')
                if not os.path.exists(output[1]):
                    with open(output[1], 'w') as out:
                        for l in allele.readlines():
                            out.write(l)
                        out.write('\n')
                else:
                    with open(output[1], 'a') as out:
                        out.write(allele.readlines()[1])
                        out.write('\n')


#!/usr/bin/python3
import os
import re
import sys
import csv
import urllib.request
import zipfile
import gzip
import json
from pyliftover import LiftOver
import haploy

import pysam
import argparse
#import snpload

in_build=37
b3x='b37'
str_db_file='str_hg19.gff3'
contig='Y'
contigmt='MT'
pos_triplet_fn = None
lo_37to38 = None
lo_38to37 = None

load_ybrowse=0

snp_by_b37 = {}

def load_ysnp_dbj(min_tree_load_level=0):
    with open('haploy_map2j.txt', 'r') as f:
        jdata = json.load(f)
        for snp in jdata['muts']:
            if b3x in snp:
                snp_by_b37[snp[b3x]] = snp

def load_ysnp_ybrowse_db():
    haploy.load_ybrowse_snp()
    global snp_by_b37
    snp_by_b37 = haploy.haplo_ybrowse_muts_by_b38
    if b3x != 'b38':
        #TODO: build conversion
        raise LookupError

#Genotype calling from pileup column
min_qual=20
min_mapqual=20
min_reads=1
read_max=100
reward_qual_threshold=999
heterozyg_ratio=0.34
ancient_mode_ends=0
def col_to_genotype(col):
    stri=''
    c=0
    for r in col.pileups:
        c+=1
        if c > read_max:
            continue
        if r.alignment.mapq < min_mapqual:
            continue
        if r.alignment.is_secondary:
            continue
        #print(r.indel)
        if r.is_del:
            #print('TODO del:', r)
            stri += 'D'
        elif r.is_refskip:
            #print('TODO skip:', r)
            stri += 'I'
        elif r.indel < 0:
            #snp db pos is offset like this
            stri += 'D'
        elif r.indel > 0:
            #snp db pos is offset like this
            stri += 'I'
        else:
            if r.alignment.qual:
                try:
                    q = ord(r.alignment.qual[r.query_position])
                    #print(r)
                except UnicodeDecodeError:
                    return '--'
            else:
                q = 255
            if q >= min_qual:
                base=r.alignment.query_sequence[r.query_position]
                if r.query_position < ancient_mode_ends or (len(r.alignment.query_sequence)-1) - r.query_position < ancient_mode_ends:
                    if r.query_position < ancient_mode_ends:
                        dist=r.query_position
                    else:
                        dist=(len(r.alignment.query_sequence)-1) - r.query_position
                    #if (base == 'T' and not r.alignment.is_reverse) or (base == 'A' and r.alignment.is_reverse):
                    #if (base == 'T' and r.alignment.is_reverse) or (base == 'A' and not r.alignment.is_reverse):
                    if (base == 'T') or (base == 'A'):
                        print('Ancient discard: ', r.query_position, len(r.alignment.query_sequence), dist, base, r.alignment.is_reverse)
                        continue
                stri += base
                if q >= reward_qual_threshold:
                    stri += base #experimental: duplicate to reward high quality bases
    #print(stri, c)
    na=stri.count('A')
    nc=stri.count('C')
    ng=stri.count('G')
    nt=stri.count('T')
    ni=stri.count('I')
    nd=stri.count('D')
    tot=na+nc+nt+ng+ni+nd
    if tot==0:
        #print('No read:', col.pos, stri)
        return '--'
    da={'A':na, 'C':nc, 'G':ng, 'T':nt, 'I':ni, 'D':nd}
    das=sorted(da.items(), key = lambda x:(x[1], x[0]), reverse=True)
    first=das[0][1]
    second=das[1][1]
    gt='--'
    hetness=0
    if tot >= min_reads:
        gt=das[0][0]+das[0][0]
        hetness = float(second)/(first+second)
        if hetness > heterozyg_ratio:
            gt=das[0][0]+das[1][0]
    #print(col.pos+1, stri, tot, das, hetness, gt)
    return gt

#interface 1based, internals 0based
def call_range(samfile, contig, pos, num, iter=None):
    ret=['--']*num
    cols = samfile.pileup(contig=contig, start=pos-1, stop=pos+num-1, min_base_quality=min_qual, min_mapping_quality=min_mapqual)
    for col in cols:
        if col.reference_pos+1 >= pos and col.reference_pos+1 < pos+num:
            if iter and not col.reference_pos+1 in iter:
                continue
            #print(iter, col.reference_pos+1)
            #print(col.get_query_sequences(), col.get_query_qualities())
            r=col_to_genotype(col)
            ret[col.reference_pos-(pos-1)]=r
    #print(ret)
    return ret

def pos_triplet_37(p):
    global contig
    global lo_37to38
    #TODO 36
    b36='0'
    b37=p
    c38=lo_37to38.convert_coordinate(contig, int(p))
    if c38:
        b38=str(c38[0][1])
    else:
        b38='0'
    return b36, b37, b38

def pos_triplet_38(p):
    global contig
    global lo_38to37
    b36='0'
    c37=lo_38to37.convert_coordinate(contig, int(p))
    #print(p, c37)
    if c37:
        b37=str(c37[0][1])
    else:
        b37='0'
    b38=p
    return b36, b37, b38

def find_ysnps(snpset, samfile):
    global contig
    global pos_triplet_fn
    num_snps=0
    binsize=10000
    binbase=1
    bin=[]
    keys=snp_by_b37.keys()
    keys=sorted(int(i) for i in keys)
    last=keys[-1]
    for pos in keys:
        #if pos > 2930000:
        #    break
        if pos < binbase+binsize:
            bin.append(pos)
        if not pos < binbase+binsize or pos==last:
            #prev bin is rdy
            print('Y', binbase, bin)
            r=call_range(samfile, contig, binbase, binsize, iter=bin)
            #print(r)
            for p in bin:
                if p-binbase < len(r):
                    b36, b37, b38 = pos_triplet_fn(p)
                    gen = r[p-binbase][0]
                    snp = {'id': 'snipsa_%d'%p,
                        'cr': '',
                        'b36': b36,
                        'b37': b37,
                        'b38': b38,
                        'gen': gen }
                    #print(p, r[p-binbase][0])
                    #print(snp)
                    snpset['Y'][p]=snp
                    if gen[0] != '-':
                        num_snps+=1

            while not pos < binbase+binsize:
                binbase+=binsize
            bin=[pos]
    return num_snps

str_by_id={}
# Source: http://www.ybrowse.org/gbrowse2/gff/
def load_ystr_db():
    global b3x
    global str_db_file
    #haplo_muts_by_b36 = pickle.load( open( "haploy_map.txt", "rb" ) )

    with open(str_db_file, 'r') as f:
        for line in f:
            if len(line) < 3:
                continue
            if line[0] == '#':
                continue
            row = line.split('\t')
            s=row[3]
            e=row[4]
            id=row[8].split(';')[0].split('=')[1]
            strep={b3x:int(s),
                'l': int(e)-int(s)+1,
                'id':id}
            #print(strep)
            str_by_id[id]=strep

#find at least 4 repetitions of 3...30 characters
#str_re = re.compile(r'(.{2,30}?)\1{3,}')
str_re = re.compile(r'([ACGT]{3,30}?)\1{6,}')
def find_ystrs(snpset, samfile):
    global b3x
    global contig
    global pos_triplet_fn
    num_snps=0
    for s in str_by_id:
        if str_by_id[s]['l'] > 500:
            continue
        p=str_by_id[s][b3x]
        seq_list = call_range(samfile, contig, p, str_by_id[s]['l'])
        seq=''
        for c in seq_list:
            seq+=c[0]
        seq=seq.replace('D','')
        #if 'D' in seq:
        #print(p, s, seq)

        b36, b37, b38 = pos_triplet_fn(p)
        snp = {'id': 'snipsa_'+s,
            'cr': 'Y',
            'b36': b36,
            'b37': b37,
            'b38': b38,
            'gen': '=0' }
        #snp[b3x]=p

        #for m in str_re.finditer(seq):
        #    num=int(len(m.group(0))/len(m.group(1)))
        #    print('M:', p, s, m.span(), num, m.group(1), seq)

        m=str_re.search(seq)
        if m:
            num=int(len(m.group(0))/len(m.group(1)))
            snp['gen']='=%d'%num
            if m.group(0)[0]=='-':
                continue
            #print(s, m.group(1), num)
            #print(s, m.group(0), num, m.group(1))
            #print(p, s, num, m.group(1), seq)
            print(s, num, m.group(1), '\t', seq)
        else:
            print(s, seq, '-')
            continue
        #print(snp)
        snpset['Y'][p]=snp
        num_snps+=1
    return num_snps

snp_by_mtpos = {}

def load_mtdb():
    with open('haplomt_map.txt', 'r') as f:
        for line in f:
            mut = eval(line)
            snp_by_mtpos[mut['p']] = mut

def find_mtsnps(snpset, samfile):
    global contigmt
    global pos_triplet_fn
    #TODO:check mt
    num_snps=0
    binsize=100
    binbase=1
    bin=[]
    keys=snp_by_mtpos.keys()
    keys=sorted(int(i) for i in keys)
    last=keys[-1]
    for pos in keys:
        #if pos > 3000:
        #    break
        if pos < binbase+binsize:
            bin.append(pos)
        if not pos < binbase+binsize or pos==last:
            #prev bin is rdy
            print('MT', binbase, bin)
            r=call_range(samfile, contigmt, binbase, binsize, iter=bin)
            #print(r)
            for p in bin:
                if p-binbase < len(r):
                    #b36, b37, b38 = pos_triplet_fn(p)
                    gen = r[p-binbase][0]
                    snp = {'id': 'snipsa_%d'%p,
                        'cr': '',
                        'b36': p,
                        'b37': p,
                        'b38': p,
                        'gen': gen }
                    #print(p, r[p-binbase][0])
                    #print(snp)
                    snpset['MT'][p]=snp
                    if gen[0] != '-':
                        num_snps+=1

            while not pos < binbase+binsize:
                binbase+=binsize
            bin=[pos]
    return num_snps

snpauto_by_b37 = {}

def load_snpauto_db():
    with open('snp_db.txt', 'r') as f:
        c=0
        cr=1
        for line in f:
            snp = eval(line)
            if snp['cr'] not in snpauto_by_b37:
                c=0
                print("CR:", snp['cr'])
                snpauto_by_b37[snp['cr']] = {}
            c+=1
            #if c>100:
            #    continue
            snpauto_by_b37[snp['cr']][snp[b3x]] = snp

    #print(snpauto_by_b37)
    for cr in snpauto_by_b37.keys():
        print('%s: %d'%(cr, len(snpauto_by_b37[cr])) )

def find_autosnps(snpset, samfile):
    #global contig
    global pos_triplet_fn
    num_snps=0
    for cr in ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', 'X']:
        if cr not in snpauto_by_b37:
            continue
        lcontig=cr
        binsize=100000
        binbase=1
        bin=[]
        keys=snpauto_by_b37[cr].keys()
        keys=sorted(int(i) for i in keys)
        last=keys[-1]
        for pos in keys:
            #if pos > 2930000:
            #    break
            if pos < binbase+binsize:
                bin.append(pos)
            if not pos < binbase+binsize or pos==last:
                #prev bin is rdy
                print(cr, binbase, bin)
                r=call_range(samfile, lcontig, binbase, binsize, iter=bin)
                #print(r)
                for p in bin:
                    if p-binbase < len(r):
                        b36, b37, b38 = pos_triplet_fn(p)
                        gen = r[p-binbase][0]
                        snp = {'id': 'snipsa_%d'%p,
                            'cr': '',
                            'b36': b36,
                            'b37': b37,
                            'b38': b38,
                            'gen': gen }
                        #print(p, r[p-binbase][0])
                        #print(snp)
                        if cr not in snpset:
                            snpset[cr] = {}
                        snpset[cr][p]=snp
                        if gen[0] != '-':
                            num_snps+=1

                while not pos < binbase+binsize:
                    binbase+=binsize
                bin=[pos]

    return num_snps

def get_build(fname):
    build=''
    samfile = pysam.AlignmentFile(fname, get_rtype(fname))
    #print(samfile.text)
    for l in samfile.text.split('\n'):
        if not l.startswith('@SQ'):
            continue
        sn = l.split('\t')[1][3:]
        #print(l.split('\t'))
        #print(sn)
        if sn == '1': build=37
        if sn == 'Y': build=37
        if sn == 'X': build=37
        if sn == 'MT': build=37
        if sn.startswith('chr'): build=38
    #print(build)
    return build

def get_rtype(fname):
    if fname.endswith('.sam'):
        return 'r'
    if fname.endswith('.bam'):
        return 'rb'
    if fname.endswith('.cram'):
        return 'rc'
    raise

def get_index_fname(fname):
    if fname.endswith('.bam'):
        return fname+'.bai'
    if fname.endswith('.cram'):
        return fname+'.crai'
    raise

def index_if_needed(samfname):
    if not os.path.exists(get_index_fname(samfname)):
        print('Creating index for %s'%samfname)
        pysam.index(samfname)

def setup_conv(in_build):
    global b3x
    global str_db_file
    global contig
    global contigmt
    global pos_triplet_fn
    global lo_37to38
    global lo_38to37
    print("Loading LiftOver conversion chain file for build %d..."%in_build)
    if in_build == 19:
        b3x='b37'
        str_db_file='str_hg19.gff3'
        contig='chrY'
        contigmt='chrM'
        pos_triplet_fn = pos_triplet_37
        lo_37to38 = LiftOver('crossmap/GRCh37_to_GRCh38.chain.gz')
    elif in_build == 37:
        b3x='b37'
        str_db_file='str_hg19.gff3'
        contig='Y'
        contigmt='MT'
        pos_triplet_fn = pos_triplet_37
        lo_37to38 = LiftOver('crossmap/GRCh37_to_GRCh38.chain.gz')
    else:
        b3x='b38'
        str_db_file='str_hg38.gff3'
        contig='chrY'
        contigmt='chrM'
        pos_triplet_fn = pos_triplet_38
        lo_38to37 = LiftOver('crossmap/GRCh38_to_GRCh37.chain.gz')

#TODO:temporary interface, redo
convert_ystr=1
convert_y=1
convert_mt=1
convert_snpauto=0
def full_convert(samfname):
    index_if_needed(samfname)
    samfile = pysam.AlignmentFile(samfname, get_rtype(samfname))

    snpset={}
    snpset['Y']={}
    snpset['MT']={}
    asnps=0
    ysnps=0
    ysrts=0
    mtnsps=0

    if convert_snpauto:
        print("Loading SNP DB...")
        load_snpauto_db()
        print("Reading SNPs...")
        asnps = find_autosnps(snpset, samfile)
        global snpauto_by_b37
        snpauto_by_b37 = {}

    if convert_ystr:
        print("Loading STR DB...")
        load_ystr_db()
        ysrts = find_ystrs(snpset, samfile)

    if convert_y:
        print("Loading Y SNP DB2...")
        if load_ybrowse:
            load_ysnp_ybrowse_db()
        else:
            load_ysnp_dbj()
        print("Reading Y SNPs...")
        ysnps = find_ysnps(snpset, samfile)

    if convert_mt:
        print("Loading MT DB...")
        load_mtdb()
        print("Reading SNPs...")
        mtnsps = find_mtsnps(snpset, samfile)

    samfile.close()

    print("Y strs: %d Y snps: %d MT snps: %d"%(ysrts, ysnps, mtnsps))

    return snpset


from dataclasses import dataclass
import requests
import time

@dataclass(init=True)
class DatabricksJob:
  # Set up the API endpoint and token
  databricks_url: str
  api_token: str
  api_endpoint: str = f"{databricks_url}/api/2.0/jobs"

  def run_job_from_id(self, job_id):
    payload = {
      "job_id": job_id
    }
    # Call the job in parallel
    post_response = requests.post(f"{api_endpoint}/run-now", headers={"Authorization": f"Bearer {api_token}"}, json=payload)
    #print(post_response.content)
    return post_response.json()["run_id"]

  def discover_job_id(self, job_name):
    get_response = requests.get(f"{api_endpoint}/list", headers={"Authorization": f"Bearer {api_token}"})
    for item in get_response.json()["jobs"]:
      if item["settings"]["name"].upper() == job_name.upper():
        return item["job_id"]
    return None

  def run_job_from_name(self, job_name):
    job_id = self.discover_job_id(job_name)
    print(f"job_id={job_id}")
    run_id = self.run_job_from_id(job_id)
    print(f"run_id={run_id}")
    return run_id

  def wait_finish(self, run_id):
    # Wait finish
    while True:
        get_response = requests.get(f"{api_endpoint}/runs/get?run_id={run_id}", headers={"Authorization": f"Bearer {api_token}"})
        #print(get_response.content)
        status = get_response.json()["state"]["life_cycle_state"]
        print(f"status={status}")
        if status in ("TERMINATED", "INTERNAL_ERROR"):
            return ""
        time.sleep(5)

# And that's all you need to run:
databricks_url = "https://xxx.azuredatabricks.net"
api_token="xxx"
job_name = "job_name"

dj = DatabricksJob(databricks_url, api_token)
run_id = dj.run_job_from_name(job_name)
dj.wait_finish(run_id)


from django.shortcuts import render, get_object_or_404, redirect
from django.contrib.auth.decorators import login_required

from .models import Post, Group, User
from .forms import PostForm
from yatube.constants import POSTS_PER_STR
from .utils import create_page_object

def index(request):
    post_list = Post.objects.all()
    page_obj = create_page_object(request, post_list, POSTS_PER_STR)

    context = {
        'page_obj': page_obj,
    }
    return render(request, 'posts/index.html', context)

def group_posts(request, slug):
    group = get_object_or_404(Group, slug=slug)
    posts = group.group_posts.select_related('group')
    page_obj = create_page_object(request, posts, POSTS_PER_STR)

    context = {
        'group': group,
        'page_obj': page_obj,
    }
    return render(request, 'posts/group_list.html', context)

def profile(request, username):
    user = get_object_or_404(User, username=username)
    posts = user.posts.select_related('author')
    posts_count = Post.objects.filter(author__exact=user).count
    page_obj = create_page_object(request, posts, POSTS_PER_STR)

    context = {
        "username": username,
        "author": user,
        "posts_count": posts_count,
        "page_obj": page_obj,
    }
    return render(request, 'posts/profile.html', context)

def post_detail(request, post_id):
    post = get_object_or_404(Post, pk=post_id)
    posts_count = Post.objects.filter(author__exact=post.author).count
    context = {
        "post": post,
        "posts_count": posts_count,
    }
    return render(request, 'posts/post_detail.html', context)

@login_required
def post_create(request):
    form = PostForm(request.POST or None)
    context = {
        'form': form,
    }

    if form.is_valid():
        form.instance.author = request.user
        form.save()
        return redirect('posts:profile', request.user)
    return render(request, "posts/create_post.html", context)

@login_required
def post_edit(request, post_id):
    # post = Post.objects.get(pk=post_id)
    post = get_object_or_404(Post, pk=post_id)
    form = PostForm(request.POST or None, instance=post)
    if post.author != request.user:
        return redirect('posts:post_detail', post.id)
    if form.is_valid():
        post = form.save()
        form.instance.author = request.user
        post.save()
        return redirect('posts:post_detail', post_id)
    context = {
        'form': form,
        'is_edit': True,
        'post': post,
    }
    return render(request, "posts/create_post.html", context)


"""
Given an n x n binary matrix grid, return the length of the shortest clear path in the matrix. If there is no clear path, return -1.

A clear path in a binary matrix is a path from the top-left cell (i.e., (0, 0)) to the bottom-right cell (i.e., (n - 1, n - 1)) such that:

All the visited cells of the path are 0.
All the adjacent cells of the path are 8-directionally connected (i.e., they are different and they share an edge or a corner).
The length of a clear path is the number of visited cells of this path.

"""

from collections import deque

class Solution:
    def shortestPathBinaryMatrix(self, grid: List[List[int]]) -> int:

        if not grid or (not grid[0]):
            return -1

        if grid[0][0] != 0:
            return -1

        queue = deque()
        start = (0,0,1)

        visied = [[False] * len(grid[0]) for _ in range(len(grid))]
        queue.append(start)
        visied[0][0] = True

        end_i, end_j = len(grid)-1, len(grid[0])-1

        while queue:
            i,j, level = queue.popleft()

            if (i,j) == (end_i, end_j):
                return level

            for i_step, j_step in [(-1,-1), (-1,0), (-1,1), (0,1), (1,1), (1,0), (1,-1), (0,-1)]:
                next_i, next_j = i + i_step, j + j_step

                if 0 <= next_i < len(grid) and 0 <= next_j < len(grid) and (not visied[next_i][next_j]) and grid[next_i][next_j] == 0:
                    queue.append((next_i, next_j, level + 1))
                    visied[next_i][next_j] = True

        return -1


from turtle import width
import pandas as pd
import numpy as np
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.cluster.hierarchy import linkage, dendrogram
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
import plotly.figure_factory as ff
from dash import Dash, html, dcc, dash_table, Input, Output, State
import dash_bootstrap_components as dbc
from plotly.tools import mpl_to_plotly

app = Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP , dbc.icons.BOOTSTRAP])

Landings = pd.read_csv("./Code/Meteorite_Landings.csv")
Impacts = pd.read_csv("./Code/impacts.csv")
Orbit = pd.read_csv("./Code/orbits.csv")

# 3d scatter lat and long

Landings["mass (g)"] = Landings["mass (g)"].fillna(0)
Fallen = Landings[(Landings["fall"] == "Fell") & (Landings["mass (g)"] > 0)]
Fallen = Fallen.sort_values("year")
def plot_3d_landings(withSize=False):
    if(withSize):
        scatter_3d_no_size = px.scatter_3d(Fallen, x='year', y='reclat', z='reclong', size="mass (g)", color='recclass')
    else:
        scatter_3d_no_size = px.scatter_3d(Fallen, x='year', y='reclat', z='reclong', color='recclass')
    return scatter_3d_no_size

def plot_map(size=None,animation_frame="year", projection="orthographic"):
    Fallen["year"] = Fallen["year"].apply(int)
    # if(withSize):
    map_plot = px.scatter_geo(Fallen, lat=Fallen.reclat,
                        lon=Fallen.reclong,
                        color="recclass",
                        hover_name="name",
                        size=size,animation_frame=animation_frame, projection=projection)
    # if(animation_frame=="year"):
    map_plot.update_layout(showlegend=False)
    # else:
        # map_plot.update_layout(showlegend=True)
    return map_plot

rel_plot = px.scatter_matrix(Impacts, dimensions=["Asteroid Velocity", "Asteroid Magnitude", "Asteroid Diameter (km)"], labels={"Asteroid Velocity":"Speed", "Asteroid Magnitude": "Magnitude", "Asteroid Diameter (km)": "Size"}, color="Possible Impacts")

histogram = make_subplots(rows=2, cols=2, subplot_titles=('Possible Impacts','Asteroid Velocity', 'Asteroid Magnitude', "Asteroid Diameter (km)" ))

histogram.add_trace(go.Histogram(x=Impacts['Possible Impacts'], name="Possible Impacts"), row=1, col=1)
histogram.add_trace(go.Histogram(x=Impacts['Asteroid Velocity'], name="Asteroid Velocity"), row=2, col=1)
histogram.add_trace(go.Histogram(x=Impacts['Asteroid Magnitude'], name="Asteroid Magnitude"), row=1, col=2)
histogram.add_trace(go.Histogram(x=Impacts['Asteroid Diameter (km)'], name="Asteroid Diameter (km)"), row=2, col=2)
histogram.update_layout(showlegend=False, title_text="Distribution of astroid features", height=1000)

pd.options.mode.chained_assignment = None
YearsPlot = Impacts[Impacts["Period End"] <= 2199 ]
YearsPlot["Start"] = YearsPlot["Period Start"].apply(str)
YearsPlot["Finish"] = YearsPlot["Period End"].apply(str)
timeline = px.timeline(YearsPlot,  x_start="Start", x_end="Finish", y="Object Name")
timeline.update_layout(title_text="Lifetime of Asteroid", height=1000)

x = Impacts.corr().columns
y = Impacts.corr().index
new = Impacts.corr().reset_index(drop=True)
corr_mat = [(Impacts.corr()[d] * 10000)//100  for d in new ]
corr_mat = np.array(corr_mat)
x= np.array(x).tolist()
y= np.array(y).tolist()
heatmap = ff.create_annotated_heatmap(corr_mat, x=x,y=y, annotation_text=corr_mat, colorscale='Viridis')

# helpers

Impacts.drop(['Maximum Torino Scale'], axis=1, inplace=True)
Features = np.array(Impacts.columns[1:]).reshape(3,3)

def drawBoxPlot(Impacts, Features):
    boxPlot = make_subplots(rows=Features.shape[0], cols=Features.shape[1])

    for iy, ix in np.ndindex(Features.shape):
        boxPlot.add_trace(go.Box(y=Impacts[Features[iy][ix]], name=Features[iy][ix]), row=iy+1, col=ix+1)

    boxPlot.update_layout(height=1000, showlegend=False)
    return boxPlot

def removeOutliers(df, features):
    df_copy = df.copy()

    for feature in features:

        # Calculate q1, q3 and iqr
        q3 = df[feature].quantile(0.75)
        q1 = df[feature].quantile(0.25)
        iqr = q3 - q1

        # Get local minimum and maximum
        local_min = q1 - (1.5 * iqr)
        local_max = q3 + (1.5 * iqr)

        # Remove the outliers
        df_copy = df_copy[(df_copy[feature] >= local_min) & (df_copy[feature] <= local_max)]
        return df_copy

Impacts_Cleaned = removeOutliers(Impacts, Features.flatten())

def sunBurnPlot(start=0,end=9999):
    fig = px.sunburst(
        Impacts_Cleaned[(Impacts_Cleaned['Period Start'] > start) & (Impacts_Cleaned['Period End'] < end)],
        path=['Period Start','Period End', 'Object Name'], values='Asteroid Diameter (km)'
    )
    return fig

def plotElbow(df,features):
    x_cord = []
    y_cord = []

    for k in range(1, 11):
        model = KMeans(n_clusters=k)
        model.fit(df[features])
        y_cord.append(model.inertia_)
        x_cord.append(k)
    fig = px.line(x=x_cord, y=y_cord, title="Clusers list")
    return fig

def plotDendogram(Impacts_Cleaned, Features):
    distance_matrix = linkage(Impacts_Cleaned[Features.flatten()], method='complete')

    fig = ff.create_dendrogram(distance_matrix)
    fig.update_layout(height=1000, width=1500)
    return fig

theme = "plotly"
def drawCluster(X, y, dim=2 , withSize=False):
    pca = PCA(dim)
    X_component = pca.fit_transform(X)
    x1 = X_component[:, 0]
    x2 = X_component[:, 1]

    if (dim == 2):
        if(withSize):
            fig= px.scatter(x=x1, y= x2, color=y, template=theme, size=X["Asteroid Diameter (km)"])
        else:
            fig= px.scatter(x=x1, y= x2, color=y, template=theme)
        return fig

    elif (dim == 3):
        x3 = X_component[:, 2]
        if(withSize):
            fig = px.scatter_3d(X_component, x=x1, y=x2, z=x3, color=y, size=X["Asteroid Diameter (km)"], template=theme)
        else:
            fig = px.scatter_3d(X_component, x=x1, y=x2, z=x3, color=y, template=theme)
        return fig

Impact_model = KMeans(n_clusters=3)
Impact_model_labels = Impact_model.fit_predict(Impacts_Cleaned[Features.flatten()])
centroids = Impact_model.cluster_centers_

Impacts_Cleaned["Labels"] = Impact_model_labels

def drawLabelledCluster(size=None):
    fig = px.scatter(Impacts_Cleaned, x="Maximum Palermo Scale", y="Possible Impacts", color="Labels",
                 size=size)

    return fig
def drawCustomizableLabelledCluster(x=None, y=None, z= None, size=None):
    fig = px.scatter_3d(Impacts_Cleaned, x=x, y=y, z=z, color="Labels",
                 size=size)

    return fig

collapse = html.Div(
[
dbc.Button(
    "View Past Landing Data",
    id="collapse-button",
    className="mb-3",
    color="primary",
    n_clicks=0,
),
dbc.Collapse(
    dash_table.DataTable(Landings.to_dict('records'), [{"name": i, "id": i} for i in Landings.columns], page_size=10),
    id="collapse",
    is_open=False,
),
]
)

collapse_future = html.Div(
[
dbc.Button(
    "Asteroids Currently orbiting Solar System",
    id="collapse-button_future",
    className="mb-3",
    color="primary",
    n_clicks=0,
),
dbc.Collapse(
    dash_table.DataTable(Impacts.to_dict('records'), [{"name": i, "id": i} for i in Impacts.columns], page_size=10),
    id="collapse_future",
    is_open=False,
),
]
)

@app.callback(
    Output("collapse", "is_open"),
    [Input("collapse-button", "n_clicks")],
    [State("collapse", "is_open")],
)
def toggle_collapse(n, is_open):
    if n:
        return not is_open
    return is_open

@app.callback(
    Output("collapse_future", "is_open"),
    [Input("collapse-button_future", "n_clicks")],
    [State("collapse_future", "is_open")],
)
def toggle_collapse_future(n, is_open):
    if n:
        return not is_open
    return is_open

tab1_content = dbc.Card(
    dbc.CardBody(
        [
html.H1('Past Landings', className="display-6"),
    html.Hr(className="my-2"),
    collapse,
    # dash_table.DataTable(Landings.to_dict('records'), [{"name": i, "id": i} for i in Landings.columns], page_size=10),

#     dcc.Checklist(
#     [' Show Respective to size of the asteroid'],id="size_display"
# ),
html.Div(
    [
         html.H1('3D Plot of year and coordinated landed',  className="display-6"),
    html.Hr(className="my-2"),
        dbc.Checklist(
            options=[
                {"label": "Show Respective to size of the asteroid", "value": 1},
                # {"label": "Flat Map", "value": 2},
                # {"label": "Show Animation Animation", "value": 3},
            ],
            value=[],
            id="size_display",
            switch=True,
        ),
    ]
),
    dcc.Graph(id='3d_scatter'),
#     dcc.Checklist(
#     ['     ', '    ', '    '],id="size_display_map"
# ),
html.Div(
    [
    html.H1('Geo location of the places where Asteroids Hit in past',  className="display-6"),
    html.Hr(className="my-2"),
        dbc.Checklist(
            options=[
                {"label": "Show Respective to size of the asteroid", "value": 1},
                {"label": "Flat Map", "value": 2},
                {"label": "Show Animation Animation", "value": 3},
            ],
            value=[3],
            id="size_display_map",
            switch=True,
        ),
    ]
),
    dcc.Graph(id='3d_map'),
        ]
    ),
    className="mt-3",
)

tab2_content = dbc.Card(
    dbc.CardBody(
        [html.H1('Future Impact Analysis',  className="display-6"),
        collapse_future,
    html.H1('Relation Plot',  className="display-6"),
    html.Hr(className="my-2"),
    dcc.Graph(figure=rel_plot),
    html.H1('Histogram',  className="display-6"),
    html.Hr(className="my-2"),
    dcc.Graph(figure=histogram),
    html.H1('Total life time of Asteroid',  className="display-6"),
    html.Hr(className="my-2"),
    dcc.Graph(figure=timeline),
    html.H1('Heat Map',  className="display-6"),
    html.Hr(className="my-2"),
    dcc.Graph(figure=heatmap),
    html.H1('Box Plot before cleaning the data where outliers are found',  className="display-6"),
    html.Hr(className="my-2"),
    dcc.Graph(figure=drawBoxPlot(Impacts=Impacts, Features=Features)),
    html.H1('Box Plot after cleaning the data where outliers are removed',  className="display-6"),
    html.Hr(className="my-2"),
    dcc.Graph(figure=drawBoxPlot(Impacts=Impacts_Cleaned, Features=Features)),
    html.H1('Various Asteroids Classified based on start and end year',  className="display-6"),
    html.Hr(className="my-2"),
    # dcc.Slider(min=min(Impacts_Cleaned['Period Start']), max=max(Impacts_Cleaned['Period End']), step=1, id='Year_select_sun_burn'),
    dcc.Graph(figure=sunBurnPlot() ,id="SunBurn")]
    ),
    className="mt-3",
)

tab3_content = dbc.Card(
    dbc.CardBody([
html.H1('Model Building:',  className="display-6"),
    html.H1('Decide Number of clusters',  className="display-6"),
    html.Hr(className="my-2"),
    dcc.Graph(figure=plotElbow(Impacts_Cleaned, Features.flatten())),
    html.H1('Dendogram Plot',  className="display-6"),
    html.Hr(className="my-2"),
    dcc.Graph(figure=plotDendogram(Impacts_Cleaned, Features.flatten())),
    html.H1('KMean Classification',  className="display-6"),
    html.Hr(className="my-2"),
    dbc.Checklist(
            options=[
                {"label": "Show Respective to size of the asteroid", "value": 1},
            ],
            value=[],
            id="kmean_cluser_with_size",
            switch=True,
        ),
    dcc.Graph(id="kmean_cluser_plotting"),
    html.H1('KMean Classification explorable',  className="display-6"),
    html.Hr(className="my-2"),
    dbc.Checklist(
            options=[
                {"label": "Show Respective to size of the asteroid", "value": 1},
            ],
            value=[],
            id="kmean_cluser_with_size_explorable",
            switch=True,
        ),
        dbc.Select(
            placeholder="Select the Feature for X axis...",
    id="x_explorable",
    options=[{"label":i, "value":i} for i in Features.flatten()],
),

        dbc.Select(
            placeholder="Select the Feature for Y axis...",
    id="y_explorable",
    options=[{"label":i, "value":i} for i in Features.flatten()],
),
        dbc.Select(
            placeholder="Select the Feature for Z axis...",
    id="z_explorable",
    options=[{"label":i, "value":i} for i in Features.flatten()],
),
    dcc.Graph(id="kmean_cluser_plotting_explorable"),
    html.H1('KMean Classification With 2 dimension PCA',  className="display-6"),
    html.Hr(className="my-2"),
    dbc.Checklist(
            options=[
                {"label": "Show Respective to size of the asteroid", "value": 1},
            ],
            value=[],
            id="cluser_with_size",
            switch=True,
        ),
    dcc.Graph(id="cluser_plotting"),
    html.H1('KMean Classification With 3 dimension PCA',  className="display-6"),
    html.Hr(className="my-2"),
    dbc.Checklist(
            options=[
                {"label": "Show Respective to size of the asteroid", "value": 1},
            ],
            value=[],
            id="cluser_with_size_3d",
            switch=True,
        ),
    dcc.Graph(id="cluser_plotting_3d")
    ]
     ),
    className="mt-3",
)

tabs = dbc.Tabs(
    [
        dbc.Tab(tab1_content, label="Past Astroid hit on Earth"),
        dbc.Tab(tab2_content, label="Future Asteroids impact on Earth"),
        dbc.Tab(tab3_content, label="Model Building"),
    ]
)

app.layout = dbc.Container([
    html.H1('Asteroid Impact on Earth', className="display-3"),
    html.Hr(className="my-3"),
    tabs

], fluid= True)

@app.callback(
    Output('kmean_cluser_plotting_explorable', 'figure'),
    Input('kmean_cluser_with_size_explorable', 'value'),
    Input('x_explorable', 'value'),
    Input('y_explorable', 'value'),
    Input('z_explorable', 'value')
    )
def kmean_update_fclassification_plot_explore(size,x,y,z):
    print(x,y,z)
    return drawCustomizableLabelledCluster(x=x,y=y,z=z,size='Asteroid Diameter (km)' if (1 in size) else None) # plot_3d_landings(withSize=(1 in size))

@app.callback(
    Output('kmean_cluser_plotting', 'figure'),
    Input('kmean_cluser_with_size', 'value'))
def kmean_update_fclassification_plot(size):
    print(size)
    return drawLabelledCluster(size='Asteroid Diameter (km)' if (1 in size) else None) # plot_3d_landings(withSize=(1 in size))

@app.callback(
    Output('cluser_plotting_3d', 'figure'),
    Input('cluser_with_size_3d', 'value'))
def update_fclassification_plot(size):
    print(size)
    return drawCluster(X=Impacts_Cleaned[Features.flatten()], y=Impacts_Cleaned["Labels"], withSize=(1 in size), dim=3) # plot_3d_landings(withSize=(1 in size))

@app.callback(
    Output('cluser_plotting', 'figure'),
    Input('cluser_with_size', 'value'))
def update_fclassification_plot(size):
    print(size)
    return drawCluster(X=Impacts_Cleaned[Features.flatten()], y=Impacts_Cleaned["Labels"], withSize=(1 in size)) # plot_3d_landings(withSize=(1 in size))

@app.callback(
    Output('3d_scatter', 'figure'),
    Input('size_display', 'value'))
def update_figure(size):
    print(size)
    return plot_3d_landings(withSize=(1 in size))

@app.callback(
    Output('3d_map', 'figure'),
    Input('size_display_map', 'value'))
def update_figure_map(selection):
    send_args = {"animation_frame": None}
    if 1 in selection: send_args["size"]=  "mass (g)"
    if 2 in selection: send_args["projection"]= "natural earth"
    if 3 in selection: send_args["animation_frame"]= "year"
    print(send_args)
    return plot_map(**send_args)

# @app.callback(
#     Output('SunBurn', 'figure')
#     # Input('Year_select_sun_burn', 'value')
#     )
# def sun_burn_plot(selection):
#     print(selection)
#     return sunBurnPlot()

if __name__ == '__main__':
    app.run_server(debug=True, port=8080)


import skimage.segmentation as seg
from skimage import io
from skimage.transform import resize
from skimage.color import rgb2hsv, rgb2gray
import numpy as np
from PIL import Image
from scipy.spatial.distance import cosine as cosine_distance

class BipedModel:
    def __init__(self):
        pass

    def extract_features(self, img_path, return_isolated=False):
        image = self.isolate_sock(img_path)
        features = self.histogram(image)

        if return_isolated:
            return features, image

        return features

    def resize(self, image):
        # first resize the image for quicker calculations
        aspect_ratio = image.shape[1] / image.shape[0]
        resized_height = 100
        resized_width = int(resized_height * aspect_ratio)
        image = resize(image, (resized_height, resized_width), anti_aliasing=False, preserve_range=True)
        return image

    def isolate_sock(self, img_path, method='morphological_chan_vese'):
        image = io.imread(img_path)
        image = self.resize(image)
        # low min size factor because we don't want the algo to drop a segment
        methods = {
            'slic': self.slic,
            'morphological_chan_vese': self.morphological_cv,
        }
        mask = methods[method](image)
        mask = mask != 0
        image = image * np.repeat(mask[:, :, np.newaxis], 3, axis=2)
        image = image.astype(int)
        return image

    def get_similarity(self, feature1, feature2, method='intersection'):
        if method == 'cosine':
            similarity = 1-cosine_distance(feature1, feature2)

        elif method == 'intersection':
            minima = np.minimum(feature1, feature2)
            maxima = np.maximum(feature1, feature2)
            similarity = np.true_divide(np.sum(minima), np.sum(maxima))
        return similarity

    def histogram(self, image, method='3D_rgb'):
        image = image.astype(int)
        # make a linear image with all the unmasked pixels
        sock = image[np.nonzero(image.mean(axis=2))]
        sock = sock[:, np.newaxis, :]
        sock = np.uint8(sock)

        if method == 'pil_histogram':
            pil_image = Image.fromarray(sock)
            histogram = pil_image.histogram()

        if method == '3D_rgb':
            sock = sock[:, 0, :]
            bins_per_dim = 17
            bin_limits = [pow(255, i/bins_per_dim)for i in range(bins_per_dim+1)]
            bin_limits[0] = 0
            bins = np.array([bin_limits, bin_limits, bin_limits])
            histogram, edges = np.histogramdd(sock, bins=bins)

        if method == 'hue':
            sock = rgb2hsv(sock)
            sock = sock[:, 0, :]
            weights = sock[:, 1] * sock[:, 2]  # value times saturation
            histogram, edges = np.histogramdd(sock[:, 0], bins=[40], weights=weights)

        return np.array(histogram).flatten()

    def slic(self, image):
        mask = seg.slic(image, n_segments=2, min_size_factor=0.01, max_size_factor=2, compactness=10)
        return mask

    def morphological_cv(self, image):
        image = rgb2gray(image)
        mask = seg.morphological_chan_vese(image, iterations=1000, init_level_set='circle')
        return mask


import plotly.graph_objects as go
import dash
from dash import dcc, html
import csv
from PIL import Image
import numpy as np

# Read data from the CSV file
data = []
with open("spacetimecuberoutes/amsnonpeakhour.csv", "r") as csvfile:
    reader = csv.DictReader(csvfile)
    for row in reader:
        data.append(row)

# Function to split latitude and longitude values from a string in "lat, long" format
def split_lat_long(lat_long_str):
    lat, long = map(float, lat_long_str.split(','))
    return lat, long

# Create lists to store data for Plotly
origin_lat_long = []
destination_lat_long = []
distances = []
driving_times = []
public_transport_times = []
transport_modes = []
route_names = []  # New list to store route names

# Extract data from the CSV and split into separate lists for Plotly
for row in data:
    o_lat, o_long = split_lat_long(row['origin_latitude'] + ',' + row['origin_longitude'])
    origin_lat_long.append((o_lat, o_long))

    d_lat, d_long = split_lat_long(row['destination_latitude'] + ',' + row['destination_longitude'])
    destination_lat_long.append((d_lat, d_long))

    distances.append(float(row['distance']))  # Assuming the CSV contains a 'distance' column
    driving_times.append(float(row['driving_time']))  # Assuming the CSV contains a 'driving_time' column
    public_transport_times.append(float(row['public_transport']))  # Assuming the CSV contains a 'public_transport' column
    transport_modes.append(int(row['transport_modes']))  # Assuming the CSV contains a 'transport_modes' column
    route_names.append(row['route_name'])  # Assuming the CSV contains a 'route_name' column

# Initialize the Dash app
app = dash.Dash(__name__)

# Read the image you want to use as the texture
im = np.asarray(Image.open("spacetimecuberoutes/bengaluru_map.jpg"))  # reading image as a numpy array
im_x, im_y, im_layers = im.shape

# Create a meshgrid of x and y values spanning the image dimensions
x, y = np.meshgrid(np.linspace(0, im_x - 1, im_x), np.linspace(0, im_y - 1, im_y))

# Create a surface mesh from the image
surface = go.Surface(
    x=x,
    y=y,
    z=np.zeros(im.shape[:2]),
    surfacecolor=im,
    showscale=False,
    colorscale='Viridis',
    opacity=0.7,
)

# Create a scatter trace for route lines and origin-destination points
traces = []
for i in range(len(data)):
    x_vals = [origin_lat_long[i][1], destination_lat_long[i][1]]
    y_vals = [origin_lat_long[i][0], destination_lat_long[i][0]]
    z_vals = [driving_times[i]] * 2
    time_color = '#8B0000' if z_vals[0] > 90 else 'green' if z_vals[0] <= 30 else '#FFD700' if z_vals[
                                                                                0] <= 60 else 'red'
    line_width = 2 + z_vals[0] / 30

    traces.append({
        'type': 'scatter3d',
        'x': x_vals,
        'y': y_vals,
        'z': z_vals,
        'mode': 'lines',
        'line': {'color': time_color, 'width': line_width},
        'hoverinfo': 'text',
        'text': 'Route: {}
Distance: {} km
Time: {} mins'.format(route_names[i], distances[i], z_vals[0])
    })

    traces.append({
        'type': 'scatter3d',
        'x': [origin_lat_long[i][1], destination_lat_long[i][1]],
        'y': [origin_lat_long[i][0], destination_lat_long[i][0]],
        'z': z_vals,
        'mode': 'markers',
        'marker': {'size': 5, 'color': ['#00B295', '#23395B']},
        'name': 'Origin-Destination Points'
    })

# Set the layout
app.layout = html.Div([
    dcc.Graph(
        id='space-time-cube',
        figure={
            'data': traces + [surface],
            'layout': {
                'title': 'Space-Time Cube',
                'scene': {
                    'xaxis_title': 'Longitude',
                    'yaxis_title': 'Latitude',
                    'zaxis_title': 'Time (mins)',
                    'zaxis': {'tickvals': list(range(0, 200, 25)), 'ticktext': [str(i) for i in range(0, 200, 25)]},
                    'xaxis': {'range': [0, im_x], 'showticklabels': False},
                    'yaxis': {'range': [0, im_y], 'showticklabels': False},
                    'camera_eye': {'x': -1.5, 'y': -2.5, 'z': 1},
                    'bgcolor': 'rgb(240, 240, 240)',
                }
            }
        },
        style={'width': '100%', 'height': '100vh'}
    )
])

if __name__ == '__main__':
    app.run_server(debug=True)


'''
UI BITS TOOLS

Creates a QtGui.QWidget object containing all the UI and functionality for
creating bits.

By: Austin Baker
Date: 9/5/2013
'''
from PySide import QtCore
from PySide import QtGui

import maya.cmds as cmds

import marigold.ui.widgets.QTWidgets as QTWidgets
import marigold.ui.qtui_resources
import marigold.utility.TransformUtility as TransformUtility
import marigold.utility.NodeUtility as NodeUtility

class UIBitsTools( QtGui.QWidget ):
    def __init__( self ):
        super( UIBitsTools, self ).__init__()

        layout = QtGui.QVBoxLayout( self )

        # Build bits controls
        # Presets
        moduleRootBtn = QTWidgets.imageButton( 'Create Module', ':/riggingUI/icons/preset_module.png', [32,32] )
        moduleRootBtn.clicked.connect( lambda a=1:self.preset_buttons(a) )

        characterRootBtn = QTWidgets.imageButton( 'Create Character', ':/riggingUI/icons/preset_character.png', [32,32] )
        characterRootBtn.clicked.connect( lambda a=2:self.preset_buttons(a) )

        # Primitive buttons.
        sphereBtn = QTWidgets.imageButton( 'Sphere', ':/riggingUI/icons/bit_sphere.png', [32,32] )
        sphereBtn.clicked.connect( self.primitive_buttons )

        boxBtn = QTWidgets.imageButton( 'Box', ':/riggingUI/icons/bit_box.png', [32,32] )
        boxBtn.clicked.connect( self.primitive_buttons )

        cylinderBtn = QTWidgets.imageButton( 'Cylinder', ':/riggingUI/icons/bit_cylinder.png', [32,32] )
        cylinderBtn.clicked.connect( self.primitive_buttons )

        coneBtn = QTWidgets.imageButton( 'Cone', ':/riggingUI/icons/bit_cone.png', [32,32] )
        coneBtn.clicked.connect( self.primitive_buttons )

        torusBtn = QTWidgets.imageButton( 'Torus', ':/riggingUI/icons/bit_torus.png', [32,32] )
        torusBtn.clicked.connect( self.primitive_buttons )

        # Bit tools.
        matchTranslationBtn = QTWidgets.imageTextButton( 'Match Translation', ':/riggingUI/icons/icon_match_translation.png', [16,16] )
        matchTranslationBtn.clicked.connect( lambda a='tran':TransformUtility.matchTransforms( a ) )

        matchRotationBtn = QTWidgets.imageTextButton( 'Match Rotation', ':/riggingUI/icons/icon_match_rotation.png', [16,16] )
        matchRotationBtn.clicked.connect( lambda a='rot':TransformUtility.matchTransforms( a ) )

        matchAllBtn = QTWidgets.imageTextButton( 'Match All', ':/riggingUI/icons/icon_match_all.png', [16,16] )
        matchAllBtn.clicked.connect( lambda a='all':TransformUtility.matchTransforms( a ) )

        copyBitSettingsBtn = QTWidgets.imageTextButton( 'Copy Bit Settings', ':/riggingUI/icons/bit_copy_settings.png', [16,16] )
        copyBitSettingsBtn.clicked.connect( lambda:NodeUtility.copyBitSettings() )

        addChildBtn = QTWidgets.imageTextButton( 'Add Child', ':/riggingUI/icons/bit_add_child.png', [16,16] )
        addChildBtn.clicked.connect( lambda:NodeUtility.setBitChild() )

        deleteChildBtn = QTWidgets.imageTextButton( 'Delete Child', ':/riggingUI/icons/bit_delete_child.png', [16,16] )
        deleteChildBtn.clicked.connect( lambda:NodeUtility.deleteBitChild() )

        # Build bits layout.
        # Presets.
        presetsHeader = QTWidgets.basicLabel( 'Bit Presets', 'bold', 14, 'white', '2B2B30' )
        presetsHeader.setMinimumHeight( 30 )
        presetsHeader.setAlignment( QtCore.Qt.AlignCenter )

        presetsFrame = QTWidgets.basicFrame()
        presetsLayout = QtGui.QHBoxLayout()
        presetsLayout.addWidget( moduleRootBtn )
        presetsLayout.addWidget( characterRootBtn )
        presetsFrame.setLayout( presetsLayout )

        # Primitive buttons
        bitPrimitivesHeader = QTWidgets.basicLabel( 'Bit Primitives', 'bold', 14, 'white', '2B2B30' )
        bitPrimitivesHeader.setMinimumHeight( 30 )
        bitPrimitivesHeader.setAlignment( QtCore.Qt.AlignCenter )

        primitiveFrame = QTWidgets.basicFrame()

        primitiveLayout = QtGui.QHBoxLayout()
        primitiveLayout.addWidget( sphereBtn )
        primitiveLayout.addWidget( boxBtn )
        primitiveLayout.addWidget( cylinderBtn )
        primitiveLayout.addWidget( coneBtn )
        primitiveLayout.addWidget( torusBtn )
        primitiveFrame.setLayout( primitiveLayout )

        # Bit tools.
        bitToolsHeader = QTWidgets.basicLabel( 'Bit Tools', 'bold', 14, 'white', '2B2B30' )
        bitToolsHeader.setMinimumHeight( 30 )
        bitToolsHeader.setAlignment( QtCore.Qt.AlignCenter )

        bitToolsGrid = QtGui.QGridLayout()
        bitToolsGrid.setColumnMinimumWidth( 0, 100 )
        bitToolsGrid.setColumnMinimumWidth( 1, 100 )
        bitToolsGrid.setColumnMinimumWidth( 2, 100 )
        bitToolsGrid.setSpacing( 2 )
        bitToolsGrid.setContentsMargins( 0,0,0,0 )
        # widget, row, col
        bitToolsGrid.addWidget( matchTranslationBtn, 0, 0 )
        bitToolsGrid.addWidget( matchRotationBtn, 0, 1 )
        bitToolsGrid.addWidget( matchAllBtn, 0, 2 )
        bitToolsGrid.addWidget( copyBitSettingsBtn, 1, 0 )
        bitToolsGrid.addWidget( addChildBtn, 1, 1 )
        bitToolsGrid.addWidget( deleteChildBtn, 1, 2 )

        # Added the bits widgets to the sub-layout of the main window.
        layout.addWidget( presetsHeader )
        layout.addWidget( presetsFrame )
        layout.addWidget( bitPrimitivesHeader )
        layout.addWidget( primitiveFrame )
        layout.addWidget( bitToolsHeader )
        layout.addLayout( bitToolsGrid )

    def preset_buttons( self, inPresetType ):
        '''
        Creates a module or character root, complete with appropriate meta nodes.

        @param inPresetType: Int. 1 is a module root. 2 is a character root.
        '''
        if inPresetType == 1:
            # Create module root.

            # Create the root bit.
            rootBit = cmds.makeGLBit( objecttype='glTorus', name='moduleRoot', color=(1.0,0.1,0.1),
                                      torSides=3, torRings=8, torOtRad=1.1, torInRad=1, drawPlace=0 )
            # Create the meta node.
            # Connect the node to the bit.
            pass
        elif inPresetType == 2:
            # Create character root.

            # Create the root bit.
            rootBit = cmds.makeGLBit( objecttype='glTorus', name='characterRoot', color=(1.0,0.1,0.1),
                                      torSides=3, torRings=8, torOtRad=3.1, torInRad=3, drawPlace=1 )

            # Create the meta node.
            # Connect the node to the bit.
            pass

    def primitive_buttons( self ):
        sender = self.sender()
        bit_name = 'gl{0}'.format( sender.text() )
        cmds.makeGLBit( objecttype=bit_name )


import json
import functools
import os.path
import logging
import tempfile
import subprocess
import ftrack_api.session

logger = logging.getLogger('com.ftrack.recipes.make-non-encode-web-playable')

# Gathered from https://en.wikipedia.org/wiki/Video_file_format
ENCODING_SUPPORTED_EXTENSIONS_VIDEO = [
    '.3g2',
    '.3gp',
    '.asf',
    '.avi',
    '.drc',
    '.flv',
    '.m2v',
    '.m4p',
    '.m4v',
    '.m4v',
    '.mkv',
    '.mng',
    '.mov',
    '.mp2',
    '.mp4',
    '.mpe',
    '.mpeg',
    '.mpg',
    '.mpv',
    '.mxf',
    '.nsv',
    '.ogg',
    '.ogv',
    '.qt',
    '.rm',
    '.rmvb',
    '.roq',
    '.svi',
    '.vob',
    '.webm',
    '.wmv',
    '.yuv',
]

# change these to match full path to the exact executables.
ffmpeg_cmd = 'ffmpeg'
ffprobe_cmd = 'ffprobe'

def exec_cmd(cmd):
    '''execute the provided *cmd*'''
    try:
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        (stdout, stderr) = process.communicate()
    except Exception:
        logger.exception('Failed to exectue command "{}"'.format(cmd[0]))
        raise

    if process.returncode:
        logger.error(
            (
                'Subprocess failed, dumping output:\n'
                '--------------- {0} -----------------\n'
                '{1}'
                '--------------------------------------'
            ).format(cmd[0], stderr)
        )
        raise EnvironmentError('Subprocess failed: %s' % cmd[0])

    return stdout

def generate_thumbnail(filepath):
    '''generate thumbnail from the given *filepath*'''
    destination = tempfile.NamedTemporaryFile(suffix='.jpg').name

    cmd = [ffmpeg_cmd]
    cmd += ['-v', 'error']
    cmd += ['-i', filepath]
    cmd += ['-ss', '0']
    cmd += ['-an', '-vframes', '1']
    cmd += ['-vcodec', 'mjpeg']
    cmd += ['-f', 'image2']
    cmd += [destination]

    exec_cmd(cmd)
    return destination

def get_info(filepath):
    '''get file information from the given *filepath*'''
    cmd = [ffprobe_cmd]
    cmd += ['-v', 'error']
    cmd += ['-print_format', 'json']
    cmd += ['-show_format']
    cmd += ['-show_streams']
    cmd += [filepath]
    result = exec_cmd(cmd)

    try:
        result = json.loads(result)
    except Exception:
        raise IOError('ffprobe failed')

    try:
        streams = result.get('streams', {})
        video_info = [
            stream for stream in streams if stream.get('codec_type') == 'video'
        ][0]
        format_info = result.get('format', {})

        frame_rates = video_info.get('r_frame_rate', '0/0').split('/')
        frame_rate = float(frame_rates[0]) / float(frame_rates[1])
    except Exception:
        frame_rate = 0

    frame_out = int(video_info.get('nb_frames', 0))
    if not frame_out:
        duration = float(format_info.get('duration', 0))
        frame_out = int(duration * frame_rate)

    meta = {
        'frameIn': 0,
        'frameOut': frame_out,
        'frameRate': frame_rate,
    }

    return meta

def callback(event, session):
    '''Non encoding make-web-playable hook.'''
    # http://ftrack-python-api.rtd.ftrack.com/en/stable/example/web_review.html?highlight=reviewable

    # run new event
    server_location = session.get('Location', ftrack_api.symbol.SERVER_LOCATION_ID)

    version_id = event['data']['versionId']
    path = event['data']['path']

    version = session.get('AssetVersion', version_id)

    # Validate that the path is an accessible file.
    if not os.path.isfile(path):
        raise ValueError('"{0}" is not a valid filepath.'.format(path))

    # Validate file is of the supported format
    _, file_extension = os.path.splitext(path)
    if file_extension not in ENCODING_SUPPORTED_EXTENSIONS_VIDEO:
        logger.error('"{0}" is not in a valid file format.'.format(path))
        return

    # disable previous event if we are in any accepted format.
    event.stop()

    # publish the file for review without re encoding.
    component = version.create_component(
        path=path, data={'name': 'ftrackreview-mp4'}, location=server_location
    )
    metadata = get_info(path)
    component['metadata']['ftr_meta'] = json.dumps(metadata)

    # generate and publish thumbnail
    thumbnail_path = generate_thumbnail(path)
    version.create_thumbnail(thumbnail_path)

    session.commit()
    logger.info('make-reviewable hook completed.')

def subscribe(session):
    '''Subscribe to events.'''
    topic = 'ftrack.connect.publish.make-web-playable'
    logger.info('Subscribing to event topic: {0!r}'.format(topic))
    # add new make web playable without encoding

    # Note, we are forcing the priority to a higher value (20)
    # than the default one (100) so we can intercept the old event
    # and stop it before it gets triggered, allowing us to override its behaviour.

    session.event_hub.subscribe(
        'topic="{0}" and source.user.username="{1}"'.format(topic, session.api_user),
        functools.partial(callback, session=session),
        priority=20,
    )

def register(session, **kw):
    '''Register plugin. Called when used as an plugin.'''
    # Validate that session is an instance of ftrack_api.Session. If not,
    # assume that register is being called from an old or incompatible API and
    # return without doing anything.
    if not isinstance(session, ftrack_api.session.Session):
        logger.debug(
            'Not subscribing plugin as passed argument {0!r} is not an '
            'ftrack_api.Session instance.'.format(session)
        )
        return

    subscribe(session)
    logger.debug('Plugin registered')


import torch
from torch.nn import CrossEntropyLoss
from dataclasses import dataclass
from online_attacks.online_algorithms import (
    create_online_algorithm,
    compute_competitive_ratio,
    OnlineParams,
    AlgorithmType,
    compute_indices,
)
from online_attacks.classifiers.mnist import (
    load_mnist_classifier,
    MnistModel,
    load_mnist_dataset,
)
from online_attacks.attacks import (
    create_attacker,
    Attacker,
    AttackerParams,
    compute_attack_success_rate,
)
from online_attacks.attacks.pgd import PGDParams
from online_attacks.datastream import datastream
from online_attacks.utils.parser import ArgumentParser
import numpy as np
import os
import torchvision
import math

@dataclass
class MnistParams:
    attacker_type: Attacker = Attacker.FGSM_ATTACK
    attacker_params: AttackerParams = AttackerParams()
    online_params: OnlineParams = OnlineParams(
        K=100, online_type=AlgorithmType.STOCHASTIC_VIRTUAL
    )
    model_dir: str = "/checkpoint/hberard/OnlineAttack/pretained_models/"

def main(args, params: MnistParams = MnistParams()):
    dataset = load_mnist_dataset(train=False)
    dataset = datastream.PermuteDataset(
        dataset, permutation=np.random.permutation(len(dataset))
    )

    target_classifier = load_mnist_classifier(
        args.model_type,
        index=0,
        model_dir=params.model_dir,
        device=args.device,
        eval=True,
    )
    source_classifier = load_mnist_classifier(
        args.model_type,
        index=1,
        model_dir=params.model_dir,
        device=args.device,
        eval=True,
    )

    criterion = CrossEntropyLoss(reduction="none")

    attacker = create_attacker(
        source_classifier, params.attacker_type, params.attacker_params
    )

    transform = datastream.Compose(
        [
            datastream.ToDevice(args.device),
            datastream.AttackerTransform(attacker),
            datastream.ClassifierTransform(target_classifier),
            datastream.LossTransform(criterion),
        ]
    )
    target_stream = datastream.BatchDataStream(
        dataset, batch_size=1000, transform=transform
    )

    transform = datastream.Compose(
        [
            datastream.ToDevice(args.device),
            datastream.AttackerTransform(attacker),
            datastream.ClassifierTransform(source_classifier),
            datastream.LossTransform(criterion),
        ]
    )
    source_stream = datastream.BatchDataStream(
        dataset, batch_size=1000, transform=transform
    )

    params.online_params.N = len(dataset)
    offline_algorithm, online_algorithm = create_online_algorithm(params.online_params)
    print("Computing indices...")
    source_online_indices, source_offline_indices = compute_indices(
        source_stream, [online_algorithm, offline_algorithm], pbar_flag=True
    )
    target_offline_indices = compute_indices(
        target_stream, [offline_algorithm], pbar_flag=True
    )[0]

    print("Computing Competitive Ratio...")
    comp_ratio = compute_competitive_ratio(
        source_online_indices, source_offline_indices
    )
    print(
        "Comp ratio source online vs target online: %.2f"
        % (comp_ratio / params.online_params.K)
    )
    comp_ratio = compute_competitive_ratio(
        source_online_indices, target_offline_indices
    )
    print(
        "Comp ratio source online vs target offline: %.2f"
        % (comp_ratio / params.online_params.K)
    )
    comp_ratio = compute_competitive_ratio(
        source_offline_indices, target_offline_indices
    )
    print(
        "Comp ratio target online vs target offline: %.2f"
        % (comp_ratio / params.online_params.K)
    )

    transform = datastream.Compose(
        [
            datastream.ToDevice(args.device),
            datastream.AttackerTransform(attacker),
            datastream.ClassifierTransform(target_classifier),
        ]
    )
    target_stream = datastream.BatchDataStream(
        dataset, batch_size=1000, transform=transform
    )

    random_indices = np.random.permutation(len(dataset))[: params.online_params.K]
    stream = target_stream.subset(random_indices)
    fool_rate = compute_attack_success_rate(stream)
    print("Attack success rate (Random): %.4f" % fool_rate)

    source_online_indices = [x[1] for x in source_online_indices]
    stream = target_stream.subset(source_online_indices)
    fool_rate = compute_attack_success_rate(stream)
    print("Attack success rate (Source Online): %.4f" % fool_rate)

    source_offline_indices = [x[1] for x in source_offline_indices]
    stream = target_stream.subset(source_offline_indices)
    fool_rate = compute_attack_success_rate(stream)
    print("Attack success rate (Source Offline): %.4f" % fool_rate)

    target_offline_indices = [x[1] for x in target_offline_indices]
    stream = target_stream.subset(target_offline_indices)
    fool_rate = compute_attack_success_rate(stream)
    print("Attack success rate (Source Offline): %.4f" % fool_rate)

    stream = datastream.BatchDataStream(dataset)
    stream = stream.subset(source_online_indices, batch_size=len(source_online_indices))
    x, target = next(stream)
    output_dir = os.path.join(args.output_dir, "img")
    os.makedirs(output_dir, exist_ok=True)
    torchvision.utils.save_image(
        x,
        os.path.join(output_dir, "clean.png"),
        nrow=int(math.sqrt(len(source_online_indices))),
    )

    x = attacker.perturb(x.to(args.device))
    torchvision.utils.save_image(
        x,
        os.path.join(output_dir, "attack.png"),
        nrow=int(math.sqrt(len(source_online_indices))),
    )

    return comp_ratio

if __name__ == "__main__":
    parser = ArgumentParser()

    # Online algorithm arguments
    parser.add_config("mnist_params", MnistParams)

    # MNIST dataset arguments
    parser.add_argument("--data_dir", default="./data")
    parser.add_argument("--test", action="store_true")
    parser.add_argument("--output_dir", default="./results")

    # MNIST classifier arguments
    parser.add_argument(
        "--model_type", type=MnistModel, default=MnistModel.MODEL_A, choices=MnistModel
    )

    # Attacker arguments
    parser.add_argument(
        "--attacker", type=Attacker, default=Attacker.PGD_ATTACK, choices=Attacker
    )

    args = parser.parse_args()
    args.device = (
        torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    )

    main(args, args.mnist_params)


"""
Blueprint related functions.

These Blueprint related functions generally align one-for-one with published API calls categorized in the Blueprint category

API v1 - https://t3n.zendesk.com/forums/20242326-Blueprint
API v2 - n/a
"""

import re
import os
import ftplib
# try:
#     from urllib.parse import urlparse
# except ImportError:
#     from urlparse import urlparse
if os.name == 'posix':  import curses
import operator
import clc

class Blueprint:

	visibility_stoi = { 'Public': 1, 'Private': 2, 'Shared': 3}
	classification_stoi = { 'System': 1, 'Script': 2, 'Scripts': 2, 'Software': 3}

	#@staticmethod
	#def List(type='All'):
	#	r = clc.v1.API.Call('post','Queue/ListQueueRequests',{'ItemStatusType': Queue.item_status_type_map[type] })
	#	if int(r['StatusCode']) == 0:  return(r['Requests'])

	@staticmethod
	def GetStatus(request_id,location=None,alias=None,silent=False):
		"""Gets the status of the specified Blueprint deployment.

		https://t3n.zendesk.com/entries/20561586-Get-Deployment-Status

		:param request_id: the int ID of the Blueprint Deployment to retrieve status for.
		:param alias: short code for a particular account.  If none will use account's default alias
		:param location: datacenter where group resides
		:param silent: disable status output when executed within CLI runtime
		"""
		if alias is None:  alias = clc.v1.Account.GetAlias()
		if location is None:  location = clc.v1.Account.GetLocation()
		r = clc.v1.API.Call('post','Blueprint/GetBlueprintStatus',{'AccountAlias': alias, 'RequestID': request_id, 'LocationAlias': location },silent=silent)
		if int(r['StatusCode']) == 0:  return(r)

	@staticmethod
	def GetPackages(classification,visibility):
		"""Gets a list of Blueprint Packages filtered by classification and visibility.

		https://t3n.zendesk.com/entries/20411357-Get-Packages

		:param classification: package type filter (System, Script, Software)
		:param visibility: package visibility filter (Public, Private, Shared)
		"""
		r = clc.v1.API.Call('post','Blueprint/GetPackages',
							{'Classification': Blueprint.classification_stoi[classification],'Visibility': Blueprint.visibility_stoi[visibility]})
		if int(r['StatusCode']) == 0:  return(r['Packages'])

	@staticmethod
	def GetAllPackages(classification):
		"""Gets a list of all Blueprint Packages with a given classification.

		https://t3n.zendesk.com/entries/20411357-Get-Packages

		:param classification: package type filter (System, Script, Software)
		"""
		packages = []
		for visibility in Blueprint.visibility_stoi.keys():
			try:
				for r in Blueprint.GetPackages(classification,visibility):  packages.append(dict(r.items()+{'Visibility':visibility}.items()))
			except:
				pass
		if len(packages):  return(packages)

	@staticmethod
	def GetAllSystemPackages():
		"""Returns list of all System packages."""
		return(Blueprint.GetAllPackages("System"))

	@staticmethod
	def GetAllScriptsPackages():
		"""Returns list of all Script packages."""
		return(Blueprint.GetAllPackages("Scripts"))

	@staticmethod
	def GetAllSoftwarePackages():
		"""Returns list of all Software packages."""
		return(Blueprint.GetAllPackages("Software"))

	@staticmethod
	def GetPendingPackages():
		"""Returns list of packages pending publishing."""
		r = clc.v1.API.Call('post','Blueprint/GetPendingPackages',{})
		if int(r['StatusCode']) == 0:  return(r['Packages'])

	@staticmethod
	def PackageUpload(package,ftp_url):
		"""Uploads specified zip package to cloud endpoint.

		See URL below for description on how to properly create a package:
		https://t3n.zendesk.com/entries/20348448-Blueprints-Script-and-Software-Package-Management

		:param package: path to zip file containing package.manifest and supporting scripts
		:param ftp_url: ftp URL including credentials of the form ftp://user:password@hostname
		"""
		#o = urlparse.urlparse(ftp_url)
		# Very weak URL checking
		#if o.scheme.lower() != "ftp":
		#	clc.v1.output.Status('ERROR',2,'Invalid FTP URL')
		#	return

		# Confirm file exists
		if not os.path.isfile(package):
			clc.v1.output.Status('ERROR',2,'Package file (%s) not found' % (package))
			return

		m = re.search("ftp://(?P<user>.+?):(?P<passwd>.+?)@(?P<host>.+)",ftp_url)
		try:
			ftp = ftplib.FTP(m.group('host'),m.group('user'),m.group('passwd'))
			file = open(package,'rb')
			filename = re.sub(".*/","",package)
			ftp.storbinary("STOR %s" % (filename),file)
			file.close()
			ftp.quit()
			clc.v1.output.Status('SUCCESS',2,'Blueprint package %s Uploaded' % (filename))
		except Exception as e:
			clc.v1.output.Status('ERROR',2,'FTP error %s: %s' % (ftp_url,str(e)))

		return({})

	@staticmethod
	def PackagePublish(package,classification,visibility,os):
		"""Publishes a Blueprint Package for use within the Blueprint Designer.

		https://t3n.zendesk.com/entries/20426453-Publish-Package

		:param package: path to zip file containing package.manifest and supporting scripts
		:param classification: package type (System, Script, Software)
		:param visibility: package visibility filter (Public, Private, Shared)
		:param os: list of ints containing Operating System template IDs
		"""
		r = clc.v1.API.Call('post','Blueprint/PublishPackage',
							{'Classification': Blueprint.classification_stoi[classification], 'Name': package, 'OperatingSystems': os,
							 'Visibility': Blueprint.visibility_stoi[visibility]})
		if int(r['StatusCode']) == 0:  return(r)

	# TODO - fix this mess
	@staticmethod
	def _DrawPublishPackageUIPosix(scr,linux_lst,windows_lst):
		scr.addstr(0,2,"Select all Operating Systems to be associated with this Blueprint Package: ")
		scr.addstr(1,2,"('q' when selection is complete)")

		max_pos = 0
		max_col1_len = 0
		pos = 2
		keys = linux_lst.keys()
		keys.sort
		for t in sorted(linux_lst.items(),key = lambda x :x[1]['Description']):
			l = t[0]
			l_f = "(%s)" % (l)
			if len(linux_lst[l]['Description'])>max_col1_len:
				max_col1_len = len(linux_lst[l]['Description'])
			pos += 1
			if linux_lst[l]['selected']:  scr.addstr(pos,1,"%s %s" % (l_f.ljust(4),linux_lst[l]['Description']),curses.A_BOLD)
			else:  scr.addstr(pos,1,"%s %s" % (l_f.ljust(4),linux_lst[l]['Description']))
		max_pos = pos
		pos = 2
		for t in sorted(windows_lst.items(),key = lambda x :x[1]['Description']):
			l = t[0]
			l_f = "(%s)" % (l)
			pos += 1
			if windows_lst[l]['selected']:  scr.addstr(pos,max_col1_len+9,"%s %s" % (l_f.ljust(4),windows_lst[l]['Description']),curses.A_BOLD)
			else:  scr.addstr(pos,max_col1_len+9,"%s %s" % (l_f.ljust(4),windows_lst[l]['Description']))

		if pos>max_pos:  max_pos = pos
		scr.addstr(max_pos+2,2,"	  ")
		scr.addstr(max_pos+2,2,"> ")
		scr.refresh()

		return(scr.getstr())

	# TODO - fix this mess
	@staticmethod
	def _DrawPublishPackageUI(linux_lst,windows_lst):
		print("\nSelect all Operating Systems to be associated with this Blueprint Package: ")
		print("'q' when selection is complete)\n")

		for items_lst in (linux_lst,windows_lst):
			for t in sorted(items_lst.items(),key = lambda x :x[1]['Description']):
				l = t[0]
				l_f = "(%s)" % (l)
				if items_lst[l]['selected']:  print( "[x] %s %s" % (l_f.ljust(4),items_lst[l]['Description']))
				else:  print( "[ ] %s %s" % (l_f.ljust(4),items_lst[l]['Description']))

		print( "\n")
		return(raw_input(" > "))

	# TODO - fix this mess
	@staticmethod
	def PackagePublishUI(package,type,visibility):
		"""Publishes a Blueprint Package for use within the Blueprint Designer after interactive OS selection.

		Interactive selection of one or more operating systems by name.

		:param package: path to zip file containing package.manifest and supporting scripts
		:param classification: package type (System, Script, Software)
		:param visibility: package visibility filter (Public, Private, Shared)
		"""
		# fetch OS list
		linux_lst = {'L': {'selected': False, 'Description': 'All Linux'}}
		windows_lst = {'W': {'selected': False, 'Description': 'All Windows'}}
		for r in clc.v1.Server.GetTemplates():
			r['selected'] = False
			if re.search("Windows",r['Description']):  windows_lst[str(r['OperatingSystem'])] = r
			elif re.search("CentOS|RedHat|Ubuntu",r['Description']):  linux_lst[str(r['OperatingSystem'])] = r

		# Get selections
		if os.name=='posix':
			scr = curses.initscr()
			curses.cbreak();
		while True:
			if os.name=='posix':  c = Blueprint._DrawPublishPackageUIPosix(scr,linux_lst,windows_lst)
			else:  c = Blueprint._DrawPublishPackageUI(linux_lst,windows_lst)
			if c.lower() == 'q':  break
			elif c.lower() == 'l':
				for l in linux_lst:  linux_lst[l]['selected'] = not linux_lst[l]['selected']
			elif c.lower() == 'w':
				for l in windows_lst:  windows_lst[l]['selected'] = not windows_lst[l]['selected']
			elif c in linux_lst:  linux_lst[c]['selected'] = not linux_lst[c]['selected']
			elif c in windows_lst:  windows_lst[c]['selected'] = not windows_lst[c]['selected']
		if os.name=='posix':
			curses.nocbreak(); curses.echo(); curses.endwin()

		# Extract selections
		ids = []
		for l in dict(linux_lst.items()+windows_lst.items()).values():
			if l['selected'] and 'OperatingSystem' in l:  ids.append(str(l['OperatingSystem']))
		clc.v1.output.Status('SUCCESS',2,'Selected operating system IDs: %s' % (" ".join(ids)))

		return(Blueprint.PackagePublish(package,type,visibility,ids))