from PyPDF2 import PdfFileWriter, PdfFileReader,PdfFileMerger
import os
import glob
import time

def remove_blank():
    files = os.listdir('temp')
    print(len(files))
    for i in range(len(files)):
        input_pdf = PdfFileReader(open(f"temp/temp{i}.pdf", "rb"))
        output_pdf = PdfFileWriter()
        output_pdf.addPage(input_pdf.getPage(0))
        with open(f"temp\_temp{i}.pdf", "wb") as output_file:
            output_pdf.write(output_file)

def remove_all_file():
    folder_path = 'temp'
    file_list = glob.glob(folder_path + '/*')
    for file_path in file_list:
        os.remove(file_path)

def merge(name):
    remove_blank()
    time.sleep(2)
    pdf_count = 0
    files = os.listdir('temp')
    pdf_count = int(len(files)/2)
    merger = PdfFileMerger()
    print(pdf_count)
    for i in range(pdf_count):
        file_name = f"temp/_temp{i}.pdf"
        merger.append(open(file_name, "rb"))
    with open(f"pdfs/{name}.pdf", "wb") as output_file:
        merger.write(output_file)
        time.sleep(2)

    time.sleep(2)

# merge("new")


from bottle import redirect, request, post
import uuid
import jwt
import time

from check_if_logged_in import check_if_logged_in
from global_values import *

@post("/new-tweet")
def new_tweet_post():
    if not check_if_logged_in():
        return redirect("/login")

    # title
    new_tweet_title = request.forms.get("new_tweet_title")

    # description
    new_tweet_description = request.forms.get("new_tweet_description")

    # can't post empty tweet without either title or description
    if not new_tweet_title:
        if not new_tweet_description:
            return redirect("/new-tweet?error=empty")

    # decode jwt cookie to get user information for tweet
    user_information = jwt.decode(request.get_cookie("jwt", secret="secret"), JWT_KEY, algorithms=["HS256"])
    user_username = user_information["username"]
    user_first_name = user_information["first_name"]
    user_id = user_information["id"]

    # append new tweet with values
    new_tweet = {
        "id": str(uuid.uuid4()),
        "user_id": user_id,
        "first_name": user_first_name,
        "username": user_username,
        "title": new_tweet_title,
        "description": new_tweet_description,
        "time_posted": time.localtime(),
        "time_edited": None,
    }
    TWEETS.append(new_tweet)

    return redirect("/dashboard")


import os
import torch
from torch.utils.data import Dataset
import torchvision

import data

DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
PGT_LOAD_FILE = "pseudo_gt_thesis.pth"
CLEAN_PGT_LOAD_FILE = "cleaned_pseudo_gt_thesis.pth"

# img size: (200,400)

class TabletopWorkDataset(Dataset):
    OBJ_ID = 3
    BB_SIZE = (560,560)
    def __init__(self, config, return_pgt=False, cleaned_pgt=True, return_gt=False, start=0, end=20000, material=True):
        """
        Dataloader for the RGBD dataset to work on the dataset using different modes:

        Arguments:
            start: Start index of the interval of images to use from the dataset
            end: End index of the interval of images to use from the dataset
            mode: Defines the mode of the dataset which determines the actions taken on the dataset:
                0: Dataset is initialized to generate pseudo ground truths
                1: Dataset is initialized to return the pseudo ground truths and images used for training
        """
        super().__init__()

        self.data_dir = data.id_to_path[config["obj_id"]] if material else data.id_to_path_uniform[config["obj_id"]]

        self.config = config

        self.meta_info = load_meta_info(self.data_dir)
        self.obj_id = self.meta_info[2]['OBJECT_ID']

        self.len = end-start
        self.start = start

        self.return_pgt = return_pgt
        self.cleaned_pgt = cleaned_pgt
        self.return_gt = return_gt

    def __getitem__(self, idx):
        # Define the frame from the given index
        idx += self.start
        frame_id = str(idx).zfill(6)
        data_frame_dir = os.path.join(self.data_dir, frame_id)
        # Load the data needed by the pose labeling scheme
        try:
            image = torch.load(os.path.join(data_frame_dir,"rgb_tensor.pt"))
            seg_data = torch.load(os.path.join(data_frame_dir, "seg_tensor.pt"))
            depth_data = torch.load(os.path.join(data_frame_dir, "depth_tensor.pt"))
            loaded=True
        except:
            try:
                meta_data = torch.load(os.path.join(data_frame_dir, "meta_data.pt"))
                image = torch.from_numpy(meta_data['rgb_tensor'][...,:3])
                seg_data = torch.from_numpy(meta_data['seg_tensor'].astype("int32"))
                depth_data = torch.from_numpy(meta_data['depth_tensor'])
                loaded=True
            except:
                image =  -torch.eye(4)
                seg_data =  -torch.eye(4)
                depth_data =  -torch.eye(4)
                print(f"Data for frame {idx} could not been loaded!")
                loaded=False

        if self.config["verbose"]:
            torchvision.utils.save_image(image.permute(2,0,1)/255., "output/pose_labeling_scheme/org_image.png")
            torchvision.utils.save_image(depth_data.unsqueeze(0), "output/pose_labeling_scheme/depth_image.png")

        #seg_mask = (seg_data==self.obj_id).int()
        #depth_data = depth_data * seg_mask
        intrinsic = torch.tensor([2/self.meta_info[0][0,0], 2/self.meta_info[0][1,1],image.shape[1]/2, image.shape[0]/2])# (fx, fy, cx, cy)

        pseudo_ground_truth = -torch.eye(4)
        ground_truth = -torch.eye(4)

        if self.return_pgt:
            try:
                if self.cleaned_pgt:
                    pseudo_ground_truth = torch.load(os.path.join(self.data_dir, frame_id, CLEAN_PGT_LOAD_FILE))
                else:
                    pseudo_ground_truth = torch.load(os.path.join(self.data_dir, frame_id, PGT_LOAD_FILE))
            except:
                pseudo_ground_truth = -torch.eye(4)

                loaded=False
        if pseudo_ground_truth is None or pseudo_ground_truth.shape[0]==0:
            pseudo_ground_truth = -torch.eye(4)
            loaded=False

        if self.return_gt:
            try:
                ground_truth = torch.load(os.path.join(self.data_dir, frame_id, "ground_truth.pt"))
            except:
                ground_truth = torch.load(os.path.join(self.data_dir, frame_id, "gt.pt"))

        return {
            "image": image,
            "seg_image": seg_data,
            "depth_image": depth_data,
            "intrinsic": intrinsic,
            "pseudo_gt": pseudo_ground_truth,
            "ground_truth": ground_truth,
            "index": idx,
            "loaded": loaded
        }

    def __len__(self):
        return self.len

def load_meta_info(data_dir):
    meta_data = torch.load(os.path.join(data_dir, "000000", "meta_data.pt"))

    # Assumption that the camera calibration is consistent for all frames
    projection_matrix = meta_data['projection_matrix']
    view_matrix = meta_data['view_matrix']

    seg_id = meta_data['seg_ids']

    return projection_matrix, view_matrix, seg_id


import bpy

bl_info = {
    "name": "Apply Modifier",
    "author": "mate.sus304",
    "version": (1, 2),
    "blender": (2, 80, 0),
    "location": "View3D > Object > Apply",
    "description": "Apply All Modifier to Mesh Object",
    "warning": "",
    "wiki_url": "",
    "tracker_url": "",
    "website":"https://sites.google.com/site/matosus304blendernotes/home",
    "category": "Object"}

######################################################
is_legacy = (bpy.app.version < (2, 80, 0))

def select_object(obj, value):
    if is_legacy:
        obj.select = value
    else:
        obj.select_set(value)

def get_active_object():
    if is_legacy:
        return bpy.context.scene.objects.active
    else:
        return bpy.context.window.view_layer.objects.active

def set_active_object(obj):
    if is_legacy:
        bpy.context.scene.objects.active = obj
    else:
        bpy.context.window.view_layer.objects.active = obj

def clear_shape_keys(Name):
    obj = get_active_object()
    if obj.data.shape_keys is None:
        return True
    obj.active_shape_key_index = len(obj.data.shape_keys.key_blocks) - 1
    while len(obj.data.shape_keys.key_blocks) > 1:
        if obj.data.shape_keys.key_blocks[obj.active_shape_key_index].name == Name:
            obj.active_shape_key_index = 0
        else:
            bpy.ops.object.shape_key_remove()
    bpy.ops.object.shape_key_remove()

def clone_object(Obj):
    tmp_obj = Obj.copy()
    tmp_obj.name = "applymodifier_tmp_%s"%(Obj.name)
    tmp_obj.data = tmp_obj.data.copy()
    tmp_obj.data.name = "applymodifier_tmp_%s"%(Obj.data.name)
    if is_legacy:
        bpy.context.scene.objects.link(tmp_obj)
    else:
        bpy.context.scene.collection.objects.link(tmp_obj)
    return tmp_obj

def delete_object(Obj):
    if Obj.data.users == 1:
        Obj.data.user_clear()
    for scn in bpy.data.scenes:
        try:
            if is_legacy:
                scn.objects.unlink(Obj)
            else:
                scn.collection.objects.unlink(Obj)
        except:
            pass

######################################################

def apply_modifier(target_object=None, target_modifiers=None):
    if target_object is None:
        obj_src = get_active_object()
    else:
        obj_src = target_object

    if not obj_src.modifiers:
        #if object has no modifier then skip
        return True

    #make single user
    if obj_src.data.users != 1:
        obj_src.data = obj_src.data.copy()

    if obj_src.data.shape_keys is None:
        #if object has no shapekeys, just apply modifier
        for x in obj_src.modifiers:
            if target_modifiers is None or x.name in target_modifiers:
                try:
                    bpy.ops.object.modifier_apply(modifier=x.name)
                except RuntimeError:
                    pass
        return True

    obj_fin = clone_object(obj_src)

    set_active_object(obj_fin)
    clear_shape_keys('Basis')

    if target_modifiers is None:
        target_modifiers = []
        for x in obj_fin.modifiers:
            if x.show_viewport:
                target_modifiers.append(x.name)

    for x in target_modifiers:
        try:
            bpy.ops.object.modifier_apply(modifier=x)
        except RuntimeError:
            pass

    flag_on_error = False
    list_skipped = []

    for i in range(1, len(obj_src.data.shape_keys.key_blocks)):
        tmp_name = obj_src.data.shape_keys.key_blocks[i].name
        obj_tmp = clone_object(obj_src)

        set_active_object(obj_tmp)
        clear_shape_keys(tmp_name)

        for x in target_modifiers:
            try:
                bpy.ops.object.modifier_apply(modifier=x)
            except RuntimeError:
                pass

        select_object(obj_tmp, True)
        set_active_object(obj_fin)
        try:
            bpy.ops.object.join_shapes()
            obj_fin.data.shape_keys.key_blocks[-1].name = tmp_name
        except:
            flag_on_error = True
            list_skipped.append(tmp_name)

        delete_object(obj_tmp)

    if flag_on_error:
        def draw(self, context):
            self.layout.label("Vertex Count Disagreement! Some shapekeys skipped.")
            for s in list_skipped:
                self.layout.label(s)

        bpy.context.window_manager.popup_menu(draw, title="Error", icon='INFO')

        return False

    tmp_name = obj_src.name
    tmp_data_name = obj_src.data.name
    obj_fin.name = tmp_name + '.tmp'

    obj_src.data = obj_fin.data
    obj_src.data.name = tmp_data_name

    for x in target_modifiers:
        obj_src.modifiers.remove(obj_src.modifiers[x])

    delete_object(obj_fin)
    set_active_object(obj_src)

class OBJECT_OT_apply_all_modifier(bpy.types.Operator):
    """Apply All Modifier to Selected Mesh Object"""
    bl_idname = "object.apply_all_modifier"
    bl_label = "Apply_All_Modifier"
    bl_options = {'REGISTER', 'UNDO'}

    def execute(self, context):
        targets = []
        for x in bpy.context.selected_objects:
            targets.append(x.name)

        bpy.ops.object.select_all(action='DESELECT')
        for x in targets:
            apply_modifier(target_object=bpy.data.objects[x])

        for x in targets:
            select_object(bpy.data.objects[x], True)

        return {'FINISHED'}

class OBJECT_OT_apply_selected_modifier(bpy.types.Operator):
    """Apply Selected Modifier to Active Mesh Object"""
    bl_idname = "object.apply_selected_modifier"
    bl_label = "Apply_Selected_Modifier"
    bl_options = {'REGISTER', 'UNDO'}

    bv: bpy.props.BoolVectorProperty(name="Booleans", description="test value", size=32)

    mod_count = 0

    @classmethod
    def poll(cls, context):
        obj = context.object
        return obj and obj.type == 'MESH'

    def execute(self, context):
        obj = get_active_object()
        objname = obj.name
        bpy.ops.object.select_all(action='DESELECT')

        str_targets = []
        for i in range(self.mod_count):
            if self.bv[i]:
                str_targets.append(bpy.data.objects[objname].modifiers[i].name)

        apply_modifier(target_object=bpy.data.objects[objname], target_modifiers=str_targets)

        select_object(obj, True)
        return {'FINISHED'}

    def invoke(self, context, event):
        wm = context.window_manager
        return wm.invoke_props_dialog(self)

    def draw(self, context):
        obj = context.object
        self.mod_count = len(obj.modifiers)

        layout = self.layout
        col = layout.column()

        for i in range(self.mod_count):
            col.prop(self, "bv", text=obj.modifiers[i].name, index=i)

# Registration

def apply_all_modifier_button(self, context):
    self.layout.operator(
        OBJECT_OT_apply_all_modifier.bl_idname,
        text="Apply All Modifier")

def apply_selected_modifier_button(self, context):
    self.layout.operator(
        OBJECT_OT_apply_selected_modifier.bl_idname,
        text="Apply Selected Modifier")

def register():
    bpy.utils.register_class(OBJECT_OT_apply_all_modifier)
    bpy.utils.register_class(OBJECT_OT_apply_selected_modifier)
    bpy.types.VIEW3D_MT_object_apply.append(apply_all_modifier_button)
    bpy.types.VIEW3D_MT_object_apply.append(apply_selected_modifier_button)

def unregister():
    bpy.utils.unregister_class(OBJECT_OT_apply_all_modifier)
    bpy.utils.unregister_class(OBJECT_OT_apply_selected_modifier)
    bpy.types.VIEW3D_MT_object_apply.remove(apply_all_modifier_button)
    bpy.types.VIEW3D_MT_object_apply.remove(apply_selected_modifier_button)

if __name__ == "__main__":
    register()


from django.contrib import admin
from .models import User, Supplier

class UserAdmin(admin.ModelAdmin):
    readonly_fields = ("last_login", "password", "phone_no", "email")
    list_display = (
        "email",
        "first_name",
        "last_name",
        "is_active",
        "created_at",
    )
    list_filter = ("is_active",)
    search_fields = ("email", "first_name", "last_name",)
    ordering = ("-created_at",)

class SupplierAdmin(admin.ModelAdmin):
    # readonly_fields = ("owner", "company_name", "company_location", "rc_number", 'government_id')
    list_display = ("owner", "company_name", "company_location", "rc_number", 'government_id', 'is_verified')
    list_filter = ("is_verified",)
    search_fields = ("company_name", "company_location", "rc_number",)

admin.site.register(User, UserAdmin)
admin.site.register(Supplier, SupplierAdmin)


import numpy as np
import os, pickle
import random
import torch
import torch.nn as nn
from collections import deque
from torch import Tensor
from torch import tensor
from time import time, sleep
import src.model_greedy

if torch.cuda.is_available():
      cuda = torch.device('cuda')
else:
      cuda = torch.device('cpu')

data_path = './outputs/'
model_path = data_path+'model.bin'
memory_path = data_path+'memory.bin'

test = []

if not 'outputs' in os.listdir():
      os.mkdir('outputs')

if not 'logs' in os.listdir():
      os.mkdir('logs')

greedy_model = src.model_greedy.Model()
greedy_model.load_state_dict(torch.load(data_path+'model_greedy.bin'))

class Model(nn.Module):
      input_shape = (46, 7, 11)
      output_shape = 4

      def __init__(self):
            super(Model, self).__init__()
            channels = 64

            self.conv1 = nn.Conv2d(self.input_shape[0], channels, (3, 3), bias=False, padding=1, padding_mode='circular')
            self.conv2 = nn.Conv2d(channels, channels, (3, 3), bias=False, padding=1, padding_mode='circular')
            self.lrelu = nn.LeakyReLU()
            self.relu = nn.ReLU()
            self.logsoftmax = nn.LogSoftmax(dim=1)

            self.flatten = nn.Flatten()
            self.ln1 = nn.Linear(channels*7*11, channels)
            self.ln2 = nn.Linear(channels, channels)
            self.ln3 = nn.Linear(channels, self.output_shape)

            self.layers = [
                  self.conv1,
                  nn.LeakyReLU(),
                  self.conv2,
                  nn.LeakyReLU(),
                  self.flatten,
                  self.ln1,
                  nn.LeakyReLU(),
                  self.ln2,
                  nn.LeakyReLU(),
                  self.ln3,
                  nn.LeakyReLU()
            ]

            self.criterion = nn.MSELoss()

      def forward(self, x):
            x = x.to(cuda)
            for l in self.layers:
                  if __name__ == '__main__' and l == self.layers[10]:
                        print("\n",l, l(x).data.cpu().detach().numpy())
                        test.append(l(x).data.cpu().detach().numpy())
                  try:
                        x = l(x)
                  except:
                        assert 0, f"something wrong with layer {l}"
            return x

      def gradientDescent(self, y_pred, y_true):
            # print(y_pred[0], y_true[0])
            self.optimizer = torch.optim.Adam(self.parameters(), lr=0.0001)
            self.optimizer.zero_grad()
            loss = self.criterion(y_pred, y_true)
            loss.backward()
            self.optimizer.step()

            # print(f"          loss={loss.item()}")

class DQN:
      def __init__(self, gamma=0.9, batch_size=256, freq=5, memory_limit=20000):
            self.model = Model()
            self.model.to(cuda)

            self.gamma = gamma
            self.memory = list()
            self.memory_limit = memory_limit
            self.batch_size = batch_size

            self.learn_freq = freq
            self.save_freq = 5000
            self.step = 0

      def remember(self, state, action, reward, done, next_state):
            self.step += 1

            self.memory.append([state.to(cuda), action.to(cuda), reward, int(done), next_state.to(cuda)])
            if done:
                  self.memory.append([state.to(cuda), action.to(cuda), reward, int(done), next_state.to(cuda)])

            while len(self.memory) > self.memory_limit:
                  self.memory.pop(0)

            size = min(self.batch_size, len(self.memory))
            if self.step % self.learn_freq == 0:
                  self.train(random.sample(self.memory, size))
            if self.step % (self.learn_freq*5) == 0:
                  self.train(self.memory[-size:])
            if self.step % self.save_freq == 0:
                  self.save()

      def train(self, memory, epochs=1):
            # print(f"    step={self.step} training started")

            for _ in range(epochs):

                  state = torch.cat([sars[0] for sars in memory], axis=0).to(cuda)
                  action = torch.cat([sars[1] for sars in memory], axis=0).to(cuda).view(-1, 4)
                  reward = torch.tensor([sars[2] for sars in memory], device=cuda).view(-1, 1)
                  done = torch.tensor([sars[3] for sars in memory], device=cuda).view(-1, 1)
                  next_state = torch.cat([sars[4] for sars in memory], axis=0).to(cuda)

                  oldQ = self.model(state)
                  targetQ = (1-done)*torch.max(self.model(next_state).detach(), axis=1, keepdim=True).values
                  targetQ = (1-action)*oldQ + action*(reward + self.gamma*targetQ)
                  # print(oldQ[-1].data)
                  # print(targetQ[-1].data)
                  # print(oldQ[done.view(-1)==1][-1].data)
                  # print(targetQ[done.view(-1)==1][-1].data)
                  self.model.gradientDescent(oldQ, targetQ)

            # print(f"    {time()-b} second")

      def save(self):
            while True:
                  try:
                        with open(model_path, 'wb') as f:
                              torch.save(self.model.state_dict(), f)
                        print("MODEL SAVED")
                  except:
                        print("Warning: model saving failed, retring to save")
                        sleep(0.1)
                        continue
                  break

            while True:
                  try:
                        with open(memory_path, 'wb') as f:
                              pickle.dump(self.memory, f)
                        print("MEMORY SAVED")
                  except:
                        print("Warning: memory saving failed, retring to save")
                        sleep(0.1)
                        continue
                  break
      def load(self, load_memory=False):
            if 'model.bin' in os.listdir(data_path):
                  while True:
                        try:
                              self.model.load_state_dict(torch.load(model_path))
                              print("MODEL LOADED")
                        except:
                              print("Warning: model loading failed, retring to load")
                              sleep(0.1)
                              continue
                        break

            if 'memory.bin' in os.listdir(data_path) and load_memory:
                  while True:
                        try:
                              with open(memory_path, 'rb') as f:
                                    self.memory = pickle.load(f)
                              print("MEMORY LOADED")
                        except:
                              print("Warning: memory loading failed, retring to load")
                              sleep(0.1)
                              continue
                        break
      def turnoff(self):
            self.save()

global Q
Q = DQN()
try:
      Q.model.load_state_dict(torch.load(model_path))
      print("MODEL LOADED")
except:
      print("NEW MODEL")

if __name__ == '__main__':
      print("model.py is working")
      inp = torch.zeros((1, 46, 7, 11))
      q = tensor(inp)
      # for i in range(41, 45):
      #       inp[0][i][:] = 1
      #       # q = torch.cat((q, inp), axis=0)
      #       p = Q.model(inp)
      #       inp[0][i][:] = 0
      #       # print(p)
      for r, c in [[2, 5], [3, 6], [4, 5], [3, 4]]:
            inp[0][12][r][c] = 1
            p = Q.model(inp)
            inp[0][12][r][c] = 0
            # print(p.data)
      # print(np.mean(np.abs(test[0]-test[1])))


from cravat.cravat_report import CravatReport
import sys
import datetime
import re
import pandas as pd
import cravat
import json
import pyreadr
import os

class Reporter(CravatReport):

    def setup (self):
        self.filenames = []
        self.filename = None
        self.filename_prefix = None
        if self.savepath == None:
            self.filename_prefix = "cravat_result"
        else:
            self.filename_prefix = self.savepath
        self.levels_to_write = self.get_standardized_module_option(
            self.confs.get("pages", ["variant"])
        )
        self.separate_header_file = (
            self.get_standardized_module_option(
                self.confs.get("separate-header-file", "true")
            )
            == True
        )
        self.zip = (
            self.get_standardized_module_option(self.confs.get("zip", "false")) == True
        )
        self.show_default_cols_only = (
            self.get_standardized_module_option(
                self.confs.get("show-default-columns-only", "true")
            )
            == True
        )
        self.cols_to_display = [
            'base__hugo',
            'base__chrom',
            'base__pos',
            'base__ref_base',
            'base__alt_base',
        ]
        self.colno_to_display_hugo = self.cols_to_display.index('base__hugo')
        self.colno_to_display_chrom = self.cols_to_display.index('base__chrom')
        self.colno_to_display_pos = self.cols_to_display.index('base__pos')
        self.colno_to_display_ref = self.cols_to_display.index('base__ref_base')
        self.colno_to_display_alt = self.cols_to_display.index('base__alt_base')
        self.colname_display_dict = {
            'base__hugo': 'group_id',
            'base__chrom': 'chr',
            'base__pos': 'pos',
            'base__ref_base': 'ref',
            'base__alt_base': 'alt',
        }
        self.display_select_columns = {}
        self.display_select_columns['variant'] = len(self.cols_to_display) > 0
        self.module_col_sep = "."
        self.colnos_to_display = {}
        self.colnames_to_display = {}
        if self.display_select_columns['variant'] == False and self.show_default_cols_only:
            db = sqlite3.connect(self.dbpath)
            c = db.cursor()
            q = f'select name from sqlite_master where name like "%_header"'
            c.execute(q)
            levels = [v[0].split("_")[0] for v in c.fetchall()]
            for level in levels:
                q = f"select col_name, col_def from {level}_header"
                c.execute(q)
                for row in c.fetchall():
                    (col_name, col_def) = row
                    col_def = json.loads(col_def)
                    if "hidden" not in col_def or col_def["hidden"] == False:
                        if col_name not in self.cols_to_display:
                            self.cols_to_display.append(col_name)
        self.headers = {}
        self.dataframe_cols = {}
        self.dataframe_colnos = {}
        self.dataframe_headers = {}
        self.colno_to_colname = {}
        self.filename_postfix = 'cfgenesis.RData'
        self.data = {}
        self.wgs_reader = cravat.get_wgs_reader('hg38')
        self.enstnov_ensgnov = {}
        data_path = os.path.dirname(os.path.abspath(__file__))
        # hugo synonyms
        f = open(os.path.join(data_path, 'data', 'hugo_synonyms.txt'))
        line = f.readline()
        toks = line.split('\t')
        app_symbol_colno = toks.index('Approved symbol')
        prev_symbols_colno = toks.index('Previous symbols')
        #alias_symbols_colno = toks.index('Alias symbols')
        self.hugo_synonyms = {}
        for line in f:
            toks = line.split('\t')
            app_symbol = toks[app_symbol_colno]
            prev_symbols = [v.strip() for v in toks[prev_symbols_colno].split(',')]
            #alias_symbols = [v.strip() for v in toks[alias_symbols_colno].split(',')]
            for symbol in prev_symbols:
                self.hugo_synonyms[symbol] = app_symbol
            #for symbol in alias_symbols:
            #    self.hugo_synonyms[symbol] = app_symbol
        f.close()
        # enst to ensg
        f = open(os.path.join(data_path, 'data', 'ensg_enst.txt'))
        for line in f:
            [ensg, enst] = line[:-1].split('\t')
            self.enstnov_ensgnov[self.remove_version(enst)] = self.remove_version(ensg)
        f.close()
        # canonical enst
        f = open(os.path.join(data_path, 'data', 'MANE.GRCh38.v0.9.summary.txt'))
        f.readline()
        self.mane_ensgnv_to_enstnv = {}
        self.mane_hugo_to_canonical_enst = {}
        self.mane_hugos = []
        self.mane_hugo_to_ensg = {}
        for line in f:
            toks = line[:-1].split('\t')
            ensg = toks[1]
            hugo = toks[3]
            #if hugo in self.hugo_synonyms:
            #    hugo = self.hugo_synonyms[hugo]
            enst = toks[7]
            ensgnv = self.remove_version(ensg)
            enstnv = self.remove_version(enst)
            self.mane_ensgnv_to_enstnv[ensgnv] = enstnv
            self.mane_hugos.append(hugo)
            self.mane_hugo_to_canonical_enst[hugo] = enst
            self.mane_hugo_to_ensg[hugo] = ensgnv
        f.close()
        # enst alen
        f = open(os.path.join(data_path, 'data', 'enst_alen.txt'))
        self.enstnv_to_alens = {}
        for line in f:
            [enst, alen] = line[:-1].split('\t')
            self.enstnv_to_alens[self.remove_version(enst)] = int(alen)
        f.close()
        # hugo to ensg
        f = open(os.path.join(data_path, 'data', 'hugo_ensg_chrom.txt'))
        self.hugo_to_ensg = {}
        self.hugo_to_chrom = {}
        self.ensg_to_chrom = {}
        for line in f:
            [hugo, ensg, chrom] = line[:-1].split('\t')
            ensg = ensg.split('.')[0]
            #if hugo in self.hugo_synonyms:
            #    hugo = self.hugo_synonyms[hugo]
            self.hugo_to_ensg[hugo] = ensg
            if hugo not in self.hugo_to_chrom:
                self.hugo_to_chrom[hugo] = []
            if ensg not in self.ensg_to_chrom:
                self.ensg_to_chrom[ensg] = chrom
            self.hugo_to_chrom[hugo].append(chrom)
        f.close()
        self.csq_consequence_to_oc_so = {
            'splice_acceptor_variant': 'splice_site_variant',
            'splice_donor_variant': 'splice_site_variant',
            'frameshift_variant': 'frameshift_elongation,frameshift_truncation'
        }
        self.no_mane_hugos = {}
        if self.filterpath is None:
            self.filter_name = None
        else:
            self.filter_name = os.path.basename(self.filterpath)
        if self.filter_name not in ['coding1.json', 'coding2.json', 'coding3.json', 'coding_noncoding_1.json', 'conding_noncoding_2.json']:
            print('\nfilter filename should be one of coding1.json, coding2.json, coding3.json, coding_noncoding_1.json, and coding_noncoding_2.json. Exiting.')
            return False

    def get_standardized_module_option(self, v):
        tv = type(v)
        if tv == str:
            if "," in v:
                v = [val for val in v.split(",") if val != ""]
        if v == "true":
            v = True
        elif v == "false":
            v = False
        return v

    def should_write_level(self, level):
        if self.levels_to_write is None:
            return True
        elif level in self.levels_to_write:
            return True
        else:
            return False

    def write_preface (self, level):
        self.level = level
        if self.should_write_level(level) == False:
            return

    def write_header (self, level):
        if self.should_write_level(level) == False:
            return
        self.headers[self.level] = []
        self.dataframe_colnos[self.level] = []
        self.dataframe_cols[self.level] = []
        self.dataframe_headers[self.level] = {}
        self.colno_to_colname[self.level] = {}
        # table columns
        for colgroup_dict in self.colinfo[self.level]['colgroups']:
            colgroup_name = colgroup_dict['name']
            minfo = cravat.admin_util.get_local_module_info(colgroup_name)
            if minfo is None:
                continue
            conf = minfo.conf
            if 'output_columns' not in conf:
                continue
            for output_dict in conf['output_columns']:
                if output_dict.get('table', False) == True:
                    colname = colgroup_name + '__' + output_dict['name']
                    if colname in self.cols_to_display:
                        self.cols_to_display.remove(colname)
                    self.dataframe_cols[self.level].append(colname)
                    self.dataframe_headers[self.level][colname] = [v['name'] for v in output_dict['table_headers']]
        colno = 0
        columns = self.colinfo[level]["columns"]
        for i in range(len(columns)):
            col = columns[i]
            colname = col['col_name']
            self.colno_to_colname[self.level][colno] = colname
            self.headers[self.level].append(colname)
            if colname in self.dataframe_cols[self.level]:
                self.dataframe_colnos[self.level].append(colno)
            if colname == 'genehancer__target_genes':
                self.colno_genehancertargetgenes = colno
            elif colname == 'base__so':
                self.colno_so = colno
            elif colname == 'base__coding':
                self.colno_coding = colno
            elif colname == 'extra_vcf_info__CSQ':
                self.colno_csq = colno
            elif colname == 'extra_vcf_info__CSQ_SYMBOL':
                self.colno_csq_symbol = colno
            elif colname == 'extra_vcf_info__CSQ_Consequence':
                self.colno_csq_consequence = colno
            elif colname == 'extra_vcf_info__CSQ_LoF':
                self.colno_csq_lofs = colno
            elif colname == 'extra_vcf_info__CSQ_Gene':
                self.colno_csq_gene = colno
            elif colname == 'extra_vcf_info__CSQ_BIOTYPE':
                self.colno_csq_biotype = colno
            elif colname == 'extra_vcf_info__CSQ_Feature':
                self.colno_csq_ensts = colno
            elif colname == 'base__transcript':
                self.colno_transcript = colno
            elif colname == 'base__all_mappings':
                self.colno_all_mappings = colno
            elif colname == 'metasvm__score':
                self.colno_metasvm_score = colno
            elif colname == 'fathmm_xf__score':
                self.colno_fathmm_xf_score = colno
            elif colname == 'sift__prediction':
                self.colno_sift_prediction = colno
            elif colname == 'lrt__lrt_pred':
                self.colno_lrt_lrt_pred = colno
            elif colname == 'polyphen2__hdiv_pred':
                self.colno_polyphen2_hdiv_pred = colno
            elif colname == 'polyphen2__hvar_pred':
                self.colno_polyphen2_hvar_pred = colno
            elif colname == 'genehancer__feature_name':
                self.colno_genehancer_feature_name = colno
            elif colname == 'ensembl_regulatory_build__region':
                self.colno_ensembl_regulatory_build_region = colno
            colno += 1
        colno = 0
        self.colnos_to_display[level] = []
        self.colnames_to_display[level] = []
        for module_col_name in self.cols_to_display:
            [module_name, col_name] = module_col_name.split('__')
            for colno in range(len(columns)):
                if columns[colno]["col_name"] == module_col_name:
                    self.colnos_to_display[level].append(colno)
                    self.colnames_to_display[level].append(self.colname_display_dict[module_col_name])
                    break
        self.data[self.level] = []

    def remove_version (self, uid):
        return uid.split('.')[0]

    def convert_csq_consequence (self, c):
        cs = []
        for tok in c.split('&'):
            cs.append(self.csq_consequence_to_oc_so.get(c, c))
        cs = '&'.join(cs)
        return cs

    def has_coding_so (self, sos):
        if 'frameshift_elongation' in sos \
                or 'frameshift_truncation' in sos \
                or 'complex_substitution' in sos \
                or 'splice_site_variant' in sos \
                or 'start_lost' in sos \
                or 'stop_gained' in sos \
                or 'stop_lost' in sos \
                or 'transcript_ablation' in sos \
                or 'inframe_insertion' in sos \
                or 'inframe_deletion' in sos \
                or 'exon_loss_variant' in sos \
                or 'missense_variant' in sos:
            return True
        else:
            return False

    def parse_mapping (self, mapping):
        [enst, _, _, sos, _, _] = mapping
        return enst, sos

    def find_canonical_mapping (self, hugo, all_mappings, canonical_enstnv):
        for mapping in all_mappings[hugo]:
            enst, sos = self.parse_mapping(mapping)
            if self.remove_version(enst) == canonical_enstnv:
                return mapping
        return None

    def parse_all_mappings_str (self, all_mappings_str):
        all_mappings_t = [v.strip() for v in all_mappings_str.split(';')]
        all_mappings = {}
        for mapping_t in all_mappings_t:
            mapping = mapping_t.split(':')
            try:
                hugo = mapping[1]
                #if hugo in self.hugo_synonyms:
                #    hugo = self.hugo_synonyms[hugo]
            except:
                print(f'#####################|\nAn exception occurred. Please contact the OpenCRAVAT team with the following information:')
                print(f'#exception: getting hugo from mapping\nall_mappings_t={all_mappings_t}')
                print(f'mapping={mapping}')
                return {}
            if hugo not in all_mappings:
                all_mappings[hugo] = []
            all_mappings[hugo].append(mapping)
        return all_mappings

    def get_canonicals(self, row, all_mappings, chrom):
        # Which hugos are in MANE and which are not.
        hugos_in_mane = []
        other_hugos = []
        csq_hugos_in_mane = []
        csq_other_hugos = []
        csq_biotypes = row[self.colno_csq_biotype]
        csq_hugos = row[self.colno_csq_symbol]
        csq_ensts = row[self.colno_csq_ensts]
        if csq_ensts is None:
            csq_ensts = []
        else:
            csq_ensts = csq_ensts.split(';')
        if csq_hugos is None:
            csq_hugos = []
        else:
            csq_hugos = csq_hugos.split(';')
        for hugo in all_mappings:
            if hugo in self.mane_hugos and hugo not in hugos_in_mane:
                hugos_in_mane.append(hugo)
            elif hugo not in other_hugos:
                other_hugos.append(hugo)
        for i in range(len(csq_hugos)):
            hugo = csq_hugos[i]
            if csq_ensts[i].startswith('ENST') == False:
                continue
            if csq_biotypes[i] != 'protein_coding':
                continue
            if hugo in self.mane_hugos and hugo not in csq_hugos_in_mane:
                csq_hugos_in_mane.append(hugo)
            elif hugo not in csq_other_hugos:
                csq_other_hugos.append(hugo)
        # ENSG and canonical ENST
        self.ensgs = {}
        canonical_ensts = {}
        canonical_enstnvs = {}
        # MANE transcript as canonical
        for hugo in hugos_in_mane:
            self.ensgs[hugo] = self.mane_hugo_to_ensg[hugo]
            enst = self.mane_hugo_to_canonical_enst[hugo]
            canonical_ensts[hugo] = enst
            canonical_enstnvs[hugo] = self.remove_version(enst)
        for hugo in csq_hugos_in_mane:
            if hugo in self.ensgs:
                continue
            self.ensgs[hugo] = self.mane_hugo_to_ensg[hugo]
            enst = self.mane_hugo_to_canonical_enst[hugo]
            canonical_ensts[hugo] = enst
            canonical_enstnvs[hugo] = self.remove_version(enst)
        for hugo in other_hugos:
            if hugo in self.hugo_to_ensg and chrom in self.hugo_to_chrom[hugo]:
                self.ensgs[hugo] = self.hugo_to_ensg[hugo]
            elif hugo in csq_hugos:
                self.ensgs[hugo] = csq_genes[csq_hugos.index(hugo)]
            else:
                print(f'ENSG ID for {hugo} was not found. Using {hugo} as group_id')
                self.ensgs[hugo] = hugo
        for hugo in csq_other_hugos:
            if hugo in self.ensgs:
                continue
            if hugo in self.hugo_to_ensg and chrom in self.hugo_to_chrom[hugo]:
                self.ensgs[hugo] = self.hugo_to_ensg[hugo]
            elif hugo in csq_hugos:
                self.ensgs[hugo] = csq_genes[csq_hugos.index(hugo)]
            else:
                print(f'ENSG ID for {hugo} was not found. Using {hugo} as group_id')
                self.ensgs[hugo] = hugo
        # Longest transcript as canonical
        for hugo in other_hugos:
            mappings = all_mappings[hugo]
            enst = mappings[0][0]
            enstnv = self.remove_version(enst)
            canonical_ensts[hugo] = enst
            canonical_enstnvs[hugo] = enstnv
            if enstnv in self.enstnv_to_alens:
                canonical_alen = self.enstnv_to_alens[enstnv]
            else:
                canonical_alen = -1
            for mapping in mappings[1:]:
                enst, sos = self.parse_mapping(mapping)
                enstnv = self.remove_version(enst)
                if enstnv in self.enstnv_to_alens:
                    alen = self.enstnv_to_alens[enstnv]
                else:
                    alen = -1
                if alen > canonical_alen:
                    canonical_alen = alen
                    canonical_ensts[hugo] = enst
                    canonical_enstnvs[hugo] = enstnv
        for hugo in csq_other_hugos:
            enst = csq_ensts[0]
            enstnv = self.remove_version(enst)
            if enst.startswith('ENST'):
                canonical_ensts[hugo] = enst
                canonical_enstnvs[hugo] = enstnv
            else:
                canonical_ensts[hugo] = None
                canonical_enstnvs[hugo] = None
            if enstnv in self.enstnv_to_alens:
                canonical_alen = self.enstnv_to_alens[enstnv]
            else:
                canonical_alen = -1
            for i in range(1, len(csq_ensts)):
                enst = csq_ensts[i]
                enstnv = self.remove_version(enst)
                if enst.startswith('ENST') == False:
                    continue
                if enstnv in self.enstnv_to_alens:
                    alen = self.enstnv_to_alens[enstnv]
                else:
                    alen = -1
                if canonical_ensts[hugo] is None or alen > canonical_alen:
                    canonical_alen = alen
                    canonical_ensts[hugo] = enst
                    canonical_enstnvs[hugo] = enstnv
        # SO for canonical transcripts
        canonical_sos = {}
        for hugo in list(set(hugos_in_mane) | set(other_hugos)):
            canonical_mapping = self.find_canonical_mapping(hugo, all_mappings, canonical_enstnvs[hugo])
            if canonical_mapping is not None:
                enst, sos = self.parse_mapping(canonical_mapping)
                canonical_sos[hugo] = sos
        csq_consequences = row[self.colno_csq_consequence]
        for hugo in list(set(csq_hugos_in_mane) | set(csq_other_hugos)):
            canonical_enstnv = canonical_enstnvs[hugo]
            for i in range(len(csq_ensts)):
                if csq_ensts[i].split('.')[0] == canonical_enstnv:
                    sos = self.convert_csq_consequence(csq_consequences[i])
                    if hugo not in canonical_sos:
                        canonical_sos[hugo] = sos
                    else:
                        canonical_sos[hugo] += ',' + sos
                    break
        return canonical_enstnvs, canonical_sos

    def get_lof_of_enstnv(self, enstnv, csq_lofs, csq_enstnvs):
        if len(csq_lofs) == 0:
            return None
        if enstnv in csq_enstnvs:
            return csq_lofs[csq_enstnvs.index(enstnv)]
        else:
            return None

    def get_all_mappings(self, row):
        all_mappings_t = row[self.colno_all_mappings]
        if all_mappings_t != '':
            all_mappings = self.parse_all_mappings_str(all_mappings_t)
        else:
            all_mappings = {}

    def get_so_of_enstnv(self, row, hugo, enstnv, csq_enstnvs, all_mappings, csq_sos):
        if hugo in all_mappings:
            for mapping in all_mappings[hugo]:
                if enstnv == mapping[3].split('.')[0]:
                    return mapping[2]
        if enstnv in csq_enstnvs:
            return self.convert_csq_consequence(csq_sos[csq_enstnvs.index(enstnv)])
        else:
            return None

    def get_csq_lofs(self, row):
        csq_lofs = row[self.colno_csq_lofs]
        if csq_lofs is None:
            return []
        else:
            return csq_lofs.split(';')

    def get_csq_enstnvs(self, row):
        csq_ensts = row[self.colno_csq_ensts]
        if csq_ensts is None:
            return []
        else:
            return [v.split('.')[0] for v in csq_ensts.split(';')]

    def run_coding1_filter(self, row, all_mappings, canonical_enstnvs, canonical_sos, csq_sos):
        csq_lofs = self.get_csq_lofs(row)
        csq_enstnvs = self.get_csq_enstnvs(row)
        metasvm_score = row[self.colno_metasvm_score]
        fathmm_xf_score = row[self.colno_fathmm_xf_score]
        group_ids = set()
        for hugo in canonical_enstnvs:
            enstnv = canonical_enstnvs[hugo]
            lof = self.get_lof_of_enstnv(enstnv, csq_lofs, csq_enstnvs)
            so = self.get_so_of_enstnv(
                row, hugo, enstnv, csq_enstnvs, all_mappings, csq_sos) # oc over vep
            if  lof == 'HC'\
                or\
                (so == 'missense_variant' and metasvm_score is not None and metasvm_score > 0)\
                or\
                (fathmm_xf_score is not None and fathmm_xf_score > 0.5 and\
                    so in ['complex_substitution',
                    'exon_loss_variant',
                    'frameshift_variant',
                    'frameshift_elongation',
                    'frameshift_truncation',
                    'inframe_insertion',
                    'inframe_deletion'
                    'missense_variant',
                    'splice_site_variant',
                    'splice_acceptor_variant',
                    'splice_donor_variant',
                    'start_lost',
                    'stop_gained',
                    'stop_lost',
                    'transcript_ablation'])\
                or\
                (so == 'synonymous_variant' and\
                    fathmm_xf_score is not None and fathmm_xf_score > 0.5):
                group_ids.add(self.ensgs[hugo])
        return group_ids

    def run_coding2_filter(self, row, all_mappings, canonical_enstnvs, canonical_sos, csq_sos):
        csq_lofs = self.get_csq_lofs(row)
        csq_enstnvs = self.get_csq_enstnvs(row)
        fathmm_xf_score = row[self.colno_fathmm_xf_score]
        sift_prediction = row[self.colno_sift_prediction]
        lrt_pred = row[self.colno_lrt_lrt_pred]
        polyphen2_hdiv_pred = row[self.colno_polyphen2_hdiv_pred]
        polyphen2_hvar_pred = row[self.colno_polyphen2_hvar_pred]
        group_ids = set()
        for hugo in canonical_enstnvs:
            enstnv = canonical_enstnvs[hugo]
            lof = self.get_lof_of_enstnv(enstnv, csq_lofs, csq_enstnvs)
            so = self.get_so_of_enstnv(row, hugo, enstnv, csq_enstnvs, all_mappings, csq_sos)
            if  (\
                    so == 'missense_variant' and\
                    sift_prediction == 'Damaging' and\
                    lrt_pred == 'Deleterious' and\
                    polyphen2_hdiv_pred is not None and 'P' in polyphen2_hdiv_pred and\
                    polyphen2_hvar_pred is not None and 'P' in polyphen2_hvar_pred
                ) or\
                (fathmm_xf_score is not None and fathmm_xf_score > 0.5 and so in [
                    'complex_substitution',
                    'exon_loss_variant',
                    'frameshift_variant',
                    'frameshift_elongation',
                    'frameshift_truncation',
                    'inframe_insertion',
                    'inframe_deletion'
                    'missense_variant',
                    'splice_site_variant',
                    'splice_acceptor_variant',
                    'splice_donor_variant',
                    'start_lost',
                    'stop_gained',
                    'stop_lost',
                    'transcript_ablation']\
                ) or\
                (\
                    so == 'synonymous_variant' and\
                    fathmm_xf_score is not None and fathmm_xf_score > 0.5\
                ) or\
                (lof == 'HC'):
                group_ids.add(self.ensgs[hugo])
        return group_ids

    def run_coding3_filter(self, row, all_mappings, canonical_enstnvs, canonical_sos, csq_sos):
        csq_lofs = self.get_csq_lofs(row)
        csq_enstnvs = self.get_csq_enstnvs(row)
        fathmm_xf_score = row[self.colno_fathmm_xf_score]
        sift_prediction = row[self.colno_sift_prediction]
        lrt_pred = row[self.colno_lrt_lrt_pred]
        polyphen2_hdiv_pred = row[self.colno_polyphen2_hdiv_pred]
        polyphen2_hvar_pred = row[self.colno_polyphen2_hvar_pred]
        group_ids = set()
        for hugo in canonical_enstnvs:
            enstnv = canonical_enstnvs[hugo]
            lof = self.get_lof_of_enstnv(enstnv, csq_lofs, csq_enstnvs)
            so = self.get_so_of_enstnv(row, hugo, enstnv, csq_enstnvs, all_mappings, csq_sos)
            if  (\
                    so == 'missense_variant' and\
                    (\
                        sift_prediction == 'Damaging' or\
                        lrt_pred == 'Deleterious' or\
                        (polyphen2_hdiv_pred is not None and 'P' in polyphen2_hdiv_pred) or\
                        (polyphen2_hvar_pred is not None and 'P' in polyphen2_hvar_pred)\
                    )\
                )\
                or\
                (fathmm_xf_score is not None and fathmm_xf_score > 0.5 and so in [
                    'complex_substitution',
                    'exon_loss_variant',
                    'frameshift_variant',
                    'frameshift_elongation',
                    'frameshift_truncation',
                    'inframe_insertion',
                    'inframe_deletion'
                    'missense_variant',
                    'splice_site_variant',
                    'splice_acceptor_variant',
                    'splice_donor_variant',
                    'start_lost',
                    'stop_gained',
                    'stop_lost',
                    'transcript_ablation']) or\
                (so == 'synonymous_variant' and\
                    fathmm_xf_score is not None and fathmm_xf_score > 0.5)\
                or\
                (lof == 'HC'):
                group_ids.add(self.ensgs[hugo])
        return group_ids

    def run_coding_noncoding_filter_1(
            self, row, all_mappings, canonical_enstnvs, canonical_sos, csq_sos):
        csq_lofs = self.get_csq_lofs(row)
        csq_enstnvs = self.get_csq_enstnvs(row)
        fathmm_xf_score = row[self.colno_fathmm_xf_score]
        metasvm_score = row[self.colno_metasvm_score]
        sift_prediction = row[self.colno_sift_prediction]
        lrt_pred = row[self.colno_lrt_lrt_pred]
        genehancer_feature_name = row[self.colno_genehancer_feature_name]
        ensembl_regulatory_build_region = row[self.colno_ensembl_regulatory_build_region]
        group_ids = set()
        for hugo in canonical_enstnvs:
            enstnv = canonical_enstnvs[hugo]
            lof = self.get_lof_of_enstnv(enstnv, csq_lofs, csq_enstnvs)
            so = self.get_so_of_enstnv(row, hugo, enstnv, csq_enstnvs, all_mappings, csq_sos)
            if  lof == 'HC'\
                or\
                (so == 'missense_variant' and metasvm_score is not None and metasvm_score > 0)\
                or\
                (fathmm_xf_score is not None and fathmm_xf_score > 0.5 and\
                    so in ['complex_substitution',
                    'exon_loss_variant',
                    'frameshift_variant',
                    'frameshift_elongation',
                    'frameshift_truncation',
                    'inframe_insertion',
                    'inframe_deletion'
                    'missense_variant',
                    'splice_site_variant',
                    'splice_acceptor_variant',
                    'splice_donor_variant',
                    'start_lost',
                    'stop_gained',
                    'stop_lost',
                    'transcript_ablation'])\
                or\
                (so == 'synonymous_variant' and\
                    fathmm_xf_score is not None and fathmm_xf_score > 0.5):
                group_ids.add(self.ensgs[hugo])
            elif genehancer_feature_name == 'Enhancer' and\
                (\
                    (fathmm_xf_score is not None and fathmm_xf_score > 0.5)\
                    or\
                    (ensembl_regulatory_build_region in [\
                        'CTCF_binding_site', 'TF_binding_site'\
                    ])\
                ):
                genehancer_target_genes = [v.split(':')[0]\
                    for v in row[self.colno_genehancertargetgenes].split(',')]
                for target in genehancer_target_genes:
                    if target.startswith('ENSG'):
                        group_ids.add(target)
            elif genehancer_feature_name == 'Promoter' and\
                (\
                    (fathmm_xf_score is not None and fathmm_xf_score > 0.5)\
                    or\
                    (ensembl_regulatory_build_region in [
                        'CTCF_binding_site', 'TF_binding_site'
                    ])\
                ):
                genehancer_target_genes = [v.split(':')[0]\
                    for v in row[self.colno_genehancertargetgenes].split(',')]
                for target in genehancer_target_genes:
                    if target.startswith('ENSG'):
                        group_ids.add(target)
            elif so is not None and 'upstream_gene_variant' in so and\
                (\
                    (fathmm_xf_score is not None and fathmm_xf_score > 0.5)\
                    or\
                    (ensembl_regulatory_build_region in [\
                        'CTCF_binding_site', 'TF_binding_site'\
                    ])\
                ):
                group_ids.add(self.ensgs[hugo])
        return group_ids

    def run_coding_noncoding_filter_2(
            self, row, all_mappings, canonical_enstnvs, canonical_sos, csq_sos):
        csq_lofs = self.get_csq_lofs(row)
        csq_enstnvs = self.get_csq_enstnvs(row)
        fathmm_xf_score = row[self.colno_fathmm_xf_score]
        metasvm_score = row[self.colno_metasvm_score]
        sift_prediction = row[self.colno_sift_prediction]
        lrt_pred = row[self.colno_lrt_lrt_pred]
        genehancer_feature_name = row[self.colno_genehancer_feature_name]
        ensembl_regulatory_build_region = row[self.colno_ensembl_regulatory_build_region]
        group_ids = set()
        for hugo in canonical_enstnvs:
            enstnv = canonical_enstnvs[hugo]
            lof = self.get_lof_of_enstnv(enstnv, csq_lofs, csq_enstnvs)
            so = self.get_so_of_enstnv(row, hugo, enstnv, csq_enstnvs, all_mappings, csq_sos)
            if  lof == 'HC'\
                or\
                (so == 'missense_variant' and metasvm_score is not None and metasvm_score > 0)\
                or\
                (fathmm_xf_score is not None and fathmm_xf_score > 0.5 and\
                    so in ['complex_substitution',
                    'exon_loss_variant',
                    'frameshift_variant',
                    'frameshift_elongation',
                    'frameshift_truncation',
                    'inframe_insertion',
                    'inframe_deletion'
                    'missense_variant',
                    'splice_site_variant',
                    'splice_acceptor_variant',
                    'splice_donor_variant',
                    'start_lost',
                    'stop_gained',
                    'stop_lost',
                    'transcript_ablation'])\
                or\
                (so == 'synonymous_variant' and\
                    fathmm_xf_score is not None and fathmm_xf_score > 0.5):
                group_ids.add(self.ensgs[hugo])
            elif genehancer_feature_name == 'Enhancer' and\
                fathmm_xf_score is not None and fathmm_xf_score > 0.5 and\
                    ensembl_regulatory_build_region in [
                    'CTCF_binding_site',
                    'TF_binding_site',
                    'enhancer',
                    'open_chromatin_region',
                    'promoter',
                    'promoter_flanking_region'
                ]:
                genehancer_target_genes = [v.split(':')[0]\
                    for v in row[self.colno_genehancertargetgenes].split(',')]
                for target in genehancer_target_genes:
                    if target.startswith('ENSG'):
                        group_ids.add(target)
            elif genehancer_feature_name == 'Promoter' and\
                fathmm_xf_score is not None and fathmm_xf_score > 0.5 and\
                    ensembl_regulatory_build_region in [
                    'CTCF_binding_site',
                    'TF_binding_site'
                    'enhancer',
                    'open_chromatin_region',
                    'promoter',
                    'promoter_flanking_region'
                ]:
                genehancer_target_genes = [v.split(':')[0]\
                    for v in row[self.colno_genehancertargetgenes].split(',')]
                for target in genehancer_target_genes:
                    if target.startswith('ENSG'):
                        group_ids.add(target)
            elif so is not None and 'upstream_gene_variant' in so and\
                fathmm_xf_score is not None and fathmm_xf_score > 0.5 and\
                    ensembl_regulatory_build_region in [
                    'CTCF_binding_site',
                    'TF_binding_site'
                    'enhancer',
                    'open_chromatin_region',
                    'promoter',
                    'promoter_flanking_region'
                ]:
                group_ids.add(self.ensgs[hugo])
        return group_ids

    def write_table_row (self, row):
        if self.should_write_level(self.level) == False:
            return
        try:
            if len(self.colnos_to_display[self.level]) > 0:
                filtered_row = [row[colno] for colno in self.colnos_to_display[self.level]]
            else:
                filtered_row = row
            chrom = filtered_row[self.colno_to_display_chrom]
            pos = int(filtered_row[self.colno_to_display_pos])
            ref = filtered_row[self.colno_to_display_ref]
            alt = filtered_row[self.colno_to_display_alt]
            if ref == '-' or alt == '-': # deletion or insertion
                chrom = filtered_row[self.colno_to_display_chrom]
                pos = pos - 1
                prev_base = self.wgs_reader.get_bases(chrom, pos).upper()
                if ref != '-' and alt == '-': # deletion
                    ref = prev_base + ref
                    alt = prev_base
                elif ref == '-' and alt != '-': # insertion
                    ref = prev_base
                    alt = prev_base + alt
                filtered_row[self.colno_to_display_pos] = pos
                filtered_row[self.colno_to_display_ref] = ref
                filtered_row[self.colno_to_display_alt] = alt
            all_mappings_t = row[self.colno_all_mappings]
            if all_mappings_t != '':
                all_mappings = self.parse_all_mappings_str(all_mappings_t)
            else:
                all_mappings = {}
            csq_consequences = row[self.colno_csq_consequence]
            if csq_consequences is None:
                csq_consequences = []
            else:
                csq_consequences = csq_consequences.split(';')
            #coding = row[self.colno_coding]
            #genehancertargetgenes = row[self.colno_genehancertargetgenes]
            # VEP annotations
            #csq = row[self.colno_csq]
            #csq_hugos = row[self.colno_csq_symbol]
            #csq_genes = row[self.colno_csq_gene]
            #csq_lofs = row[self.colno_csq_lofs]
            #csq_biotypes = row[self.colno_csq_biotype]
            #csq_ensts = row[self.colno_csq_ensts]
            #if csq_hugos is None:
            #    csq_hugos = []
            #else:
                #csq_hugos = [self.hugo_synonyms[v] if v in self.hugo_synonyms else v\
                    #for v in csq_hugos.split(';')]
            #    csq_hugos = csq_hugos.split(';')
            #if csq_ensts is None:
            #    csq_ensts = []
            #else:
            #    csq_ensts = csq_ensts.split(';')
            #if csq_genes is not None:
            #    csq_genes = [m for v in csq_genes.split(',') for m in v.split(';')]
            #else:
            #    csq_toks = csq.split('|')
            #    for tok in csq_toks:
            #        if 'ENSG' in tok:
            #            csq_genes = [m for v in tok.split(',') for m in v.split(';')]
            #            break
            #if csq_genes is None:
            #    csq_genes = []
            #if csq_lofs is not None:
            #    csq_lofs = csq_lofs.split(';')
            #else:
            #    csq_toks = csq.split('|')
            #    for tok in csq_toks:
            #        if 'HC' in tok:
            #            csq_lofs = tok.split(';')
            #            break
            #if csq_lofs is None:
            #    csq_lofs = []
            #if csq_biotypes is None:
            #    csq_biotypes = []
            #else:
            #    csq_biotypes = csq_biotypes.split(';')
            canonical_enstnvs, canonical_sos = self.get_canonicals(row, all_mappings, chrom)
            # Filters
            if self.filter_name.startswith('coding1.json'):
                group_ids = self.run_coding1_filter(
                    row,
                    all_mappings,
                    canonical_enstnvs,
                    canonical_sos,
                    csq_consequences
                )
            elif self.filter_name.startswith('coding2.json'):
                group_ids = self.run_coding2_filter(
                    row,
                    all_mappings,
                    canonical_enstnvs,
                    canonical_sos,
                    csq_consequences
                )
            elif self.filter_name.startswith('coding3.json'):
                group_ids = self.run_coding3_filter(
                    row,
                    all_mappings,
                    canonical_enstnvs,
                    canonical_sos,
                    csq_consequences
                )
            elif self.filter_name.startswith('coding_noncoding_1.json'):
                group_ids = self.run_coding_noncoding_filter_1(
                    row,
                    all_mappings,
                    canonical_enstnvs,
                    canonical_sos,
                    csq_consequences
                )
            elif self.filter_name.startswith('coding_noncoding_2.json'):
                group_ids = self.run_coding_noncoding_filter_2(
                    row,
                    all_mappings,
                    canonical_enstnvs,
                    canonical_sos,
                    csq_consequences
                )
            '''
            # GeneHancer targets
            if genehancertargetgenes is not None:
                genehancertargetgenes = [v.split(':')[0].strip() for v in genehancertargetgenes.split(',')]
                for target in genehancertargetgenes:
                    if target.startswith('ENSG') and target not in group_ids:
                        group_ids.add(target)
                        genehancer_target_exists = True
                    elif target in self.hugo_to_ensg and target in self.hugo_to_chrom and chrom in self.hugo_to_chrom[target]:
                        ensg = self.hugo_to_ensg[target]
                        if ensg not in group_ids:
                            group_ids.add(ensg)
                            genehancer_target_exists = True
            '''
            wrong_chrom_ensgs = []
            for ensg in group_ids:
                if ensg in self.ensg_to_chrom and self.ensg_to_chrom[ensg] != chrom:
                    wrong_chrom_ensgs.append(ensg)
            if len(wrong_chrom_ensgs) > 0:
                print(f'@@@ wrong_chrom_ensgs={wrong_chrom_ensgs}')
            for ensg in wrong_chrom_ensgs:
                    group_ids.remove(ensg)
            '''
            so_ignores = [
                'intron_variant',
                'synonymous_variant',
                '3_prime_UTR_variant',
                '5_prime_UTR_variant',
                'downstream_gene_variant',
                'intergenic_variant',
                'non_coding_transcript_exon_variant',
                'splice_region_variant',
                'start_retained_variant',
                'stop_retained_variant',
                'mature_miRNA_variant',
                'NMD_transcript_variant',
                'non_coding_transcript_variant',
                'TFBS_ablation',
                'TFBS_amplification',
                'TF_binding_site_variant',
                'regulatory_region_ablation',
                'regulatory_region_amplification',
                'feature_elongation',
                'regulatory_region_variant',
                'feature_truncation',
                'incomplete_terminal_codon_variant',
            ]
            # Collects group_id.
            group_ids = set()
            ## coding and splice site variant
            for hugo in canonical_enstsnv:
                if hugo == '': # For example, ENSTR.
                    continue
                sos = None
                if hugo not in canonical_sos:
                    canonical_enstnv = canonical_enstnvs[hugo]
                    for i in range(len(csq_ensts)):
                        enstnv = self.remove_version(csq_ensts[i])
                        if enstnv == canonical_enstnv:
                            csq_consq = csq_consequences[i]
                            if ('intron' in csq_consq and not ('splice_donor' in csq_consq or 'splice_acceptor' in csq_consq)) or 'downstream' in csq_consq or 'non_coding' in csq_consq or 'upstream' in csq_consq:
                                break
                            elif enstnv not in self.enstnv_to_alens:
                                print(f'{enstnv} not in oc aalen')
                                break
                            elif (self.filter_name == 'coding1' or self.filter_name == 'coding2' or self.filter_name == 'coding3'):
                                if csq_biotypes[i] != 'protein_coding':
                                    break
                                elif self.has_coding_so(
                            if csq_hugos[i] in canonical_sos:
                                sos = canonical_sos[csq_hugos[i]]
                                break
                            for cano_hugo, cano_enstnv in canonical_enstnvs.items():
                                if cano_enstnv == canonical_enstnv and cano_hugo in canonical_sos:
                                    sos = canonical_sos[cano_hugo]
                                    break
                            if sos is None:
                                print(f'##################\nAn exception occurred. Please contact the OpenCRAVAT team with the following information:')
                                print(f'#exception: sos is None\n#row={row}\ncanonical_enstnvs={canonical_enstnvs}\ncanonical_sos={canonical_sos}\nin mane? {hugo in self.mane_hugos}\nall_mappings={all_mappings}\ncsq_ensts={csq_ensts}\ncsq_hugos={csq_hugos}\ncsq_consequenced={csq_consequences}\ncsq={csq}\nhugo={hugo}')
                                return
                        if sos is not None:
                            break
                    if sos is None:
                        continue
                else:
                    sos = canonical_sos[hugo]
                if self.has_coding_so(sos):
                    ensg = self.ensgs[hugo]
                    group_ids.add(ensg)
            ## HC Lof from VEP
            if len(csq_ensts) == len(csq_lofs):
                for hugo in canonical_enstsnv:
                    canonical_enstnv = canonical_enstsnv[hugo]
                    if canonical_enstnv is None:
                        continue
                    ensg = self.ensgs[hugo]
                    if ensg in group_ids:
                        continue
                    for i in range(len(csq_lofs)):
                        enst = csq_ensts[i]
                        enstnv = self.remove_version(enst)
                        lof = csq_lofs[i]
                        biotype = csq_biotypes[i]
                        ### LoF HC and BIOTYPE relationship from chr22.sqlite:
                        # frameshift_variant	protein_coding
                        # frameshift_variant&splice_region_variant	protein_coding
                        # frameshift_variant&start_lost	protein_coding
                        # frameshift_variant&stop_lost	protein_coding
                        # frameshift_variant&stop_retained_variant	protein_coding
                        # splice_acceptor_variant	protein_coding
                        # splice_acceptor_variant&coding_sequence_variant	protein_coding
                        # splice_acceptor_variant&coding_sequence_variant&intron_variant	protein_coding
                        # splice_acceptor_variant&intron_variant	protein_coding
                        # splice_donor_variant	protein_coding
                        # splice_donor_variant&coding_sequence_variant	protein_coding
                        # splice_donor_variant&coding_sequence_variant&intron_variant	protein_coding
                        # splice_donor_variant&intron_variant	protein_coding
                        # stop_gained	protein_coding
                        # stop_gained&frameshift_variant	protein_coding
                        # stop_gained&inframe_insertion	protein_coding
                        # stop_gained&inframe_insertion&splice_region_variant	protein_coding
                        # stop_gained&splice_region_variant	protein_coding
                        ### thus, no need for checking BIOTYPE "protein_coding".
                        if enstnv == canonical_enstnv and lof == 'HC':
                            group_ids.add(ensg)
                            break
            ## GeneHancer
            genehancer_target_exists = False
            if genehancertargetgenes is not None:
                genehancertargetgenes = [v.split(':')[0].strip() for v in genehancertargetgenes.split(',')]
                for target in genehancertargetgenes:
                    if target.startswith('ENSG') and target not in group_ids:
                        group_ids.add(target)
                        genehancer_target_exists = True
                    elif target in self.hugo_to_ensg and target in self.hugo_to_chrom and chrom in self.hugo_to_chrom[target]:
                        ensg = self.hugo_to_ensg[target]
                        if ensg not in group_ids:
                            group_ids.add(ensg)
                            genehancer_target_exists = True
            ## 5k upstream
            upstream_but_no_canonical = False
            if len(csq_consequences) > 0:
                for hugo in canonical_enstsnv:
                    ensg = self.ensgs[hugo]
                    if ensg in group_ids:
                        continue
                    canonical_enstnv = canonical_enstnvs[hugo]
                    for i in range(len(csq_genes)):
                        hugo = csq_hugos[i]
                        #if hugo in self.hugo_synonyms:
                        #    hugo = self.hugo_synonyms[hugo]
                        ensg = csq_genes[i]
                        enst = csq_ensts[i]
                        enstnv = self.remove_version(enst)
                        consequence = csq_consequences[i]
                        if hugo == '': # ENSR for example
                            continue
                        if 'upstream_gene_variant' in consequence:
                            if enstnv == canonical_enstnv:
                                group_ids.add(csq_genes[i])
                                upstream_but_no_canonical = False
                                break
                            else:
                                upstream_but_no_canonical = True
            if len(group_ids) == 0:
                errmsgs = set()
                correct_so = False
                for hugo in canonical_sos:
                    sos = canonical_sos[hugo].split(',')
                    if self.has_coding_so(sos):
                        correct_so = True
                        break
                if correct_so == False:
                    errmsgs.add(f'no valid so in canonical transcript')
                if genehancertargetgenes is not None \
                        and len(genehancertargetgenes) > 0 \
                        and genehancer_target_exists == False:
                    errmsgs.add(f'GeneHancer targets are not ENSG')
                if upstream_but_no_canonical:
                    errmsgs.add('5k upstream on non-canonical transcript')
                if len(csq_ensts) == 0:
                    errmsgs.add('no transcript detected')
                if 'HC' in csq_lofs:
                    correct_lof_canonical_so = False
                    for lof_i in range(len(csq_lofs)):
                        lof = csq_lofs[lof_i]
                        enst = csq_ensts[lof_i]
                        consequence = csq_consequences[lof_i]
                        hugo = csq_hugos[lof_i]
                        if hugo in canonical_enstnvs:
                            canonical = canonical_enstnvs[hugo]
                        else:
                            canonical = ''
                        if lof == 'HC' and enst.split('.')[0] == canonical\
                                and consequence not in so_ignores:
                            correct_lof_canonical_so = True
                            break
                    if correct_lof_canonical_so == False:
                        errmsgs.add('no HC lof for canonical transcript with valid so')
                no_canonical_enst = True
                for hugo in canonical_enstnvs:
                    if hugo in all_mappings:
                        mappings = all_mappings[hugo]
                        for mapping in mappings:
                            enstnv = mapping[0].split('.')[0]
                            if enstnv == canonical_enstnvs[hugo]:
                                no_canonical_enst = False
                                break
                if len(csq_hugos) > 0:
                    for enst_i in range(len(csq_ensts)):
                        enstnv = csq_ensts[enst_i].split('.')[0]
                        if enstnv.startswith('ENST') == False:
                            continue
                        hugo = csq_hugos[enst_i]
                        #if hugo in self.hugo_synonyms:
                        #    hugo = self.hugo_synonyms[hugo]
                        try:
                            if enstnv == canonical_enstnvs[hugo]:
                                no_canonical_enst = False
                                break
                        except:
                            print(f'hugo={hugo} canonical_enstnvs={canonical_enstnvs}')
                            print(f'csq_hugos={csq_hugos}')
                            print(f'row={row}')
                            raise
                if no_canonical_enst:
                    errmsgs.add('no canonical transcript')
                if len(errmsgs) == 0:
                    print(f'#################\nAn exception occurred. Please contact the OpenCRAVAT team with the following information:')
                    print(f'#exception: No gene name for {chrom} {pos} {ref} {alt}\n#row={row}\n# csq={csq}\n# row={row}\n# csq_genes={csq_genes}\n# canonical_sos={canonical_sos}\n# coding={coding}\n# csq_lofs={csq_lofs}\n# genehancertargetgenes={genehancertargetgenes}\n# csq_ensts={csq_ensts}\n# csq_consequence={csq_consequences}\n# group_ids={group_ids}\n# canonical_ensts={canonical_ensts}\n# all_mappings={all_mappings}\n# genehancer_target_exists={genehancer_target_exists}\n# errmsgs={errmsgs}')
            else:
                if chrom.startswith('chr'):
                    chrom = chrom[3:]
                    filtered_row[self.colno_to_display_chrom] = chrom
                group_ids = list(group_ids)
                group_ids.sort()
                group_ids = [v for v in group_ids if v != '']
                for group_id in group_ids:
                    filtered_row[self.colno_to_display_hugo] = group_id
                    self.data[self.level].append([v for v in list(filtered_row)])
            '''
            if chrom.startswith('chr'):
                chrom = chrom[3:]
                filtered_row[self.colno_to_display_chrom] = chrom
            group_ids = list(group_ids)
            group_ids.sort()
            group_ids = [v for v in group_ids if v != '']
            for group_id in group_ids:
                filtered_row[self.colno_to_display_hugo] = group_id
                self.data[self.level].append([v for v in list(filtered_row)])
        except Exception as e:
            print(f'#################\nAn exception occurred. Please contact the OpenCRAVAT team with the following information:')
            print(f'#exception: {e}')
            import traceback
            traceback.print_exc(file=sys.stdout)
            print(f'#row={row}')

    def end (self):
        self.dfs = {}
        for level in self.headers.keys():
            level_data = pd.DataFrame(self.data[level], columns=self.colnames_to_display[level])
            level_data = level_data.drop_duplicates()
            self.filename = f'{self.filename_prefix}.{level}.{self.filename_postfix}'
            self.filenames.append(self.filename)
            if len(level_data) > 0:
                pyreadr.write_rdata(self.filename, level_data, df_name=f'{self.filename_prefix}_{level}')
            else:
                wf = open(self.filename, 'w')
                wf.close()
        return self.filenames

def main ():
    reporter = Reporter(sys.argv)
    reporter.run()

if __name__ == '__main__':
    main()


from sklearn.ensemble import RandomForestClassifier
from scipy import signal
import pandas as pd
import numpy as np
import statsmodels.api as sm
from sklearn.preprocessing import MinMaxScaler
from statsmodels.tsa.stattools import adfuller
import pickle
from io import BytesIO

def classifier(data_set):
    with open('models/frequency1.pkl', 'rb') as f:
        frequency = pickle.load(f)
    with open('models/model1.pkl', 'rb') as f:
        model = pickle.load(f)
    with open('models/rfc_tsm1.pkl', 'rb') as f:
        rfc = pickle.load(f)
    print('Model Loaded')

    df = pd.read_csv(BytesIO(data_set))
    df=df.drop('Unnamed: 0',axis=1)
    df['point_timestamp'] = pd.to_datetime(df['point_timestamp'])
    df = df.set_index(['point_timestamp'])
    df = df.fillna(df.mean())
    indexed_df = df.copy(deep=True)
    scaler = MinMaxScaler()
    df['point_value']=scaler.fit_transform(df[['point_value']])

    dftest = adfuller(df['point_value'], autolag = "AIC")
    trend = np.polyfit(df.index.astype(int), df['point_value'], 1)[0]
    acf_1 = sm.tsa.stattools.acf(df['point_value'], nlags=1)[1]
    volatility = np.std(df['point_value'])
    freq = pd.infer_freq(df.index)

    frequencies, spectrum = signal.periodogram(df['point_value'])
    max_index = spectrum.argmax()
    cyclicity = 1 / frequencies[max_index]

    feature_value = {'Trend': trend,
                    'Autocorrelation at lag 1' : acf_1,
                    'Volatility' : volatility,
                    'Frequency' : freq,
                    'Stationarity': dftest[1],
                    'Cyclicity': cyclicity}
    if not feature_value['Frequency']:
        feature_value['Frequency'] = frequency['H']
    else:
        feature_value['Frequency'] = frequency[feature_value['Frequency']]

    pred = rfc.predict(pd.DataFrame(feature_value, index=[0]).values.reshape(1, -1))

    final_model=""
    for key, value in model.items():
        if value == pred:
            final_model = key
            break

    return indexed_df,final_model


import os, sys
from http import HTTPStatus
from fastapi import FastAPI
from fastapi import Response
from fastapi_sqlalchemy import DBSessionMiddleware
from dotenv import load_dotenv

from app.main.adapters import fast_api_adapter
from app.domain.usecases import CreateUserParams, CreateUserResponse
from app.main.factories import create_user_factory
from app.main.routes.helpers import HandledError

BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
load_dotenv(os.path.join(BASE_DIR, ".env"))

app = FastAPI()

app.add_middleware(DBSessionMiddleware, db_url=os.environ["DATABASE_URL"])

ROUTES_TAGS = ['User']

@app.get('/hello-world')
def hello_world():
    return {'hello': 'world'}

@app.post(
    '/user',
    responses={
        HTTPStatus.CREATED.value: {
            'model': CreateUserResponse
        },
        HTTPStatus.BAD_REQUEST.value: {
            'model': HandledError, 'description': 'Company or tenant not found'
        }
    },
    status_code=HTTPStatus.CREATED,
    tags=ROUTES_TAGS
)
def create_user(body: CreateUserParams, response: Response):
    request = {'body': body, 'headers': None, 'query': None}
    result = fast_api_adapter(request, create_user_factory())
    response.status_code = result.status_code
    return result.body


# youtube/youtube_api.py

import google.auth
from google.auth.transport.requests import AuthorizedSession
from google.oauth2.credentials import Credentials
from googleapiclient.discovery import build

from config.config import YOUTUBE_API_KEY

class YoutubeAPI:
    def __init__(self):
        self.credentials = None
        self.client = None

    def authenticate(self):
        self.credentials, _ = google.auth.default(scopes=['https://www.googleapis.com/auth/youtube.readonly'])
        self.client = build('youtube', 'v3', credentials=self.credentials)

    def get_latest_short_videos(self, max_results=10):
        if not self.client:
            self.authenticate()

        request = self.client.search().list(
            part='id,snippet',
            q='#shorts',
            type='video',
            maxResults=max_results,
            order='date'
        )

        response = request.execute()
        videos = response['items']
        video_ids = [video['id']['videoId'] for video in videos]
        video_urls = [f'https://www.youtube.com/watch?v={video_id}' for video_id in video_ids]

        return video_urls


from django.shortcuts import render

from django.http.response import JsonResponse
from rest_framework.parsers import JSONParser
from rest_framework import status

from api.models import SunExposure
from api.serializers import SunExposureSerializer
from rest_framework.decorators import api_view

# Create your views here.
@api_view(['GET', 'POST', 'DELETE'])
def sunexposure_list(request):
    if request.method == 'GET':
        sunexposures = SunExposure.objects.all()

        garden_id = request.GET.get('title', None)
        if garden_id is not None:
            sunexposures = sunexposures.filter(garden_id__icontains=garden_id)

        sunexposures_serializer = SunExposureSerializer(sunexposures, many=True)
        return JsonResponse(sunexposures_serializer.data, safe=False)

    elif request.method == 'POST':
        sunexposure_data = JSONParser().parse(request)
        sunexposure_serializer = SunExposureSerializer(data=sunexposure_data)
        if sunexposure_serializer.is_valid():
            sunexposure_serializer.save()
            return JsonResponse(sunexposure_serializer.data, status=status.HTTP_201_CREATED)
        return JsonResponse(sunexposure_serializer.errors, status=status.HTTP_400_BAD_REQUEST)
    elif request.method == 'DELETE':
        sunexposures = SunExposure.objects.all()

        garden_id = request.GET.get('title', None)
        if garden_id is not None:
            sunexposures = sunexposures.filter(garden_id__icontains=garden_id)

        count = sunexposures.delete()
        return JsonResponse({'message': '{} SunExposures were deleted successfully!'.format(count[0])}, status=status.HTTP_200_OK)

@api_view(['GET', 'DELETE'])
def sunexposure_detail(request, pk):
    try:
        sunexposure = SunExposure.objects.get(pk=pk)
    except SunExposure.DoesNotExist:
        return JsonResponse({'message': 'The SunExposure does not exist'}, status=status.HTTP_404_NOT_FOUND)

    if request.method == 'GET':
        sunexposure_serializer = SunExposureSerializer(sunexposure)
        return JsonResponse(sunexposure_serializer.data)
    elif request.method == 'DELETE':
        sunexposure.delete()
        return JsonResponse({'message': 'SunExposure was deleted successfully!'}, status=status.HTTP_200_OK)


import argparse
import gc
import logging
import os
import glob
import pandas as pd

import sys
sys.path.append("../ddn/")
sys.path.append("./")

from collections import defaultdict

import torch
import warnings
warnings.filterwarnings('ignore')

import numpy as np
torch.backends.cudnn.benchmark = True

from matplotlib import pyplot as plt
import matplotlib as mpl
import matplotlib.patches as patches
from matplotlib import pyplot as plt
from scipy.linalg import block_diag
from torch.utils.data import Dataset, DataLoader
#from bernstein import bernstesin_coeff_order10_new

from argoverse.map_representation.map_api import ArgoverseMap
from argoverse.data_loading.argoverse_forecasting_loader import ArgoverseForecastingLoader
from argoverse.visualization.visualize_sequences import viz_sequence
avm = ArgoverseMap()

num = 10

data_path="/datasets/argoverse/val/data"
output_dir="../results/"
t_obs=20
dt=0.3
t_obs=20
pred=False
pred_array=None
batch_size = 512
dpi=50
w,h=200,200
paths = glob.glob(os.path.join(data_path, "*.csv"))
color = {
    'polygon': '#e6cf93',
    'polygon-outline': '#e6cf93',
    'centerline': '#fceec7',
    'agent': 'blue',
    'av': 'grey',
    'other': 'grey',
    'outline': 'black'
}

avm = ArgoverseMap()

def denoise(gt_x, gt_y, w = 7):
    # denoising
    gt_x_t = []
    gt_y_t = []
    for iq in range(len(gt_x)):
        if iq >= w and iq + w <= len(gt_x):
            gt_x_t.append(np.mean(gt_x[iq: iq + w]))
            gt_y_t.append(np.mean(gt_y[iq: iq + w]))
        elif iq < w:
            okx = np.mean(gt_x[w: w + w])
            gt_x_t.append(gt_x[0] + (okx - gt_x[0]) * (iq) / w)
            oky = np.mean(gt_y[w: w + w])
            gt_y_t.append(gt_y[0] + (oky - gt_y[0]) * (iq) / w)
        else:
            okx = np.mean(gt_x[len(gt_x) - w:len(gt_x) - w  + w])
            oky = np.mean(gt_y[len(gt_x) - w: len(gt_x) - w + w])
            gt_x_t.append(okx + (gt_x[-1] - okx) * (w - (len(gt_x) - iq)) / w)
            gt_y_t.append(oky + (gt_y[-1] - oky) * (w - (len(gt_y) - iq)) / w)

    gt_x = gt_x_t
    gt_y = gt_y_t
    return gt_x, gt_y

for idx in range(len(paths)):
    path = paths[idx]
    dff = pd.read_csv(path)

    city = dff['CITY_NAME'].values[0]

    agent_df = dff[dff['OBJECT_TYPE'] == 'AGENT']
    x_a = agent_df['X'].values
    y_a = agent_df['Y'].values
    x_a, y_a = denoise(x_a, y_a)

    av_df = dff[dff['OBJECT_TYPE'] == 'AV']
    x_av = av_df['X'].values
    y_av = av_df['Y'].values
    x_av, y_av = denoise(x_av, y_av)

    others_df = dff[dff['OBJECT_TYPE'] == 'OTHERS']
    others_dfs = np.array([v for k, v in others_df.groupby('TRACK_ID')], dtype=object)
    x_o = {}
    y_o = {}

    for other_df in others_dfs:
        x_other, y_other = other_df['X'].values, other_df['Y'].values
        x_other, y_other = denoise(x_other, y_other)
        x_o[other_df['TRACK_ID'].values[0]] = x_other
        y_o[other_df['TRACK_ID'].values[0]] = other_df['Y'].values

    # group by timestamp
    dfs = [x for _, x in dff.groupby('TIMESTAMP')]

    for ind, df in enumerate(dfs):
        agent_df = df[df['OBJECT_TYPE'] == 'AGENT']
        others_df = df[df['OBJECT_TYPE'] == 'OTHERS']
        others_dfs = [x for _, x in others_df.groupby('TRACK_ID')]
#         others_dfs = np.array([v for k, v in others_df.groupby('TRACK_ID')], dtype=object)
        av_df = df[df['OBJECT_TYPE'] == 'AV']

        # agent
        x_traj = agent_df['X'].values
        y_traj = agent_df['Y'].values

        offsets = [x_a[0], y_a[0]] # offsets for other agents

        fig = plt.figure(figsize=(200/dpi,200/dpi), dpi=dpi)
        # fig = plt.figure(figsize=(10, 10), dpi=dpi)

        x_off = 75
        y_off = 75
        points = np.array([[x_a[20] - x_off, y_a[20] + y_off],[x_a[20] + x_off, y_a[20] + y_off], [x_a[20] + x_off, y_a[20] - y_off],[x_a[20] - x_off, y_a[20] - y_off],[x_a[20] - x_off, y_a[20] + y_off]])
        plt.fill(points[:, 0], points[:, 1], color=color['outline'], zorder=0)
        if ind < len(dfs) - 1:
            x_off = 0.75
            y_off = 1.25
            points = np.array([[x_traj[0] - x_off, y_traj + y_off],[x_traj[0] + x_off, y_traj + y_off], [x_traj[0] + x_off, y_traj - y_off],[x_traj[0] - x_off, y_traj - y_off],[x_traj[0] - x_off, y_traj + y_off]])

            theta = np.arctan2((y_a[ind + 1] - y_a[ind]) , (x_a[ind + 1] - x_a[ind])) - np.pi/2
            w = np.zeros(points.shape)
            A = np.matrix([[np.cos(theta), -np.sin(theta)],[np.sin(theta), np.cos(theta)]])
            points = points - np.array([x_traj[0], y_traj[0]])
            for i,v in enumerate(points): w[i] = A @ points[i]
            plt.fill(w[:, 0] + x_traj[0], w[:, 1] + y_traj[0], color=color['agent'], zorder=5)
        plt.scatter(x_traj[0], y_traj[0], color=color['agent'], label='end observed', zorder=5)

        # av
        x_traj = av_df['X'].values
        y_traj = av_df['Y'].values
        x_max, y_max = np.max(x_traj), np.max(y_traj)
        x_min, y_min = np.min(x_traj), np.min(y_traj)

        if ind < len(dfs) - 1:
            x_off = 0.75
            y_off = 1.25
            points = np.array([[x_traj[0] - x_off, y_traj + y_off],[x_traj[0] + x_off, y_traj + y_off], [x_traj[0] + x_off, y_traj - y_off],[x_traj[0] - x_off, y_traj - y_off],[x_traj[0] - x_off, y_traj + y_off]])
            theta = np.arctan2((y_av[ind + 1] - y_av[ind]) , (x_av[ind + 1] - x_av[ind])) - np.pi/2
            w = np.zeros(points.shape)
            A = np.matrix([[np.cos(theta), -np.sin(theta)],[np.sin(theta), np.cos(theta)]])
            points = points - np.array([x_traj[0], y_traj[0]])
            for i,v in enumerate(points): w[i] = A @ points[i]
            plt.fill(w[:, 0] + x_traj[0], w[:, 1] + y_traj[0], color=color['av'], zorder=4)

        plt.scatter(x_traj[-1], y_traj[-1], color=color['av'], zorder=4)

#         # others
        for indoo, other in enumerate(others_dfs):
            x_traj = other['X'].values
            y_traj = other['Y'].values
            indo = other['TRACK_ID'].values[0]
            if ind < len(dfs) - 1 and ind < len(x_o[indo]) - 1 and ind < len(y_o[indo]) - 1:
                x_off = 0.75
                y_off = 1.25
                points = np.array([[x_traj[0] - x_off, y_traj + y_off],[x_traj[0] + x_off, y_traj + y_off], [x_traj[0] + x_off, y_traj - y_off],[x_traj[0] - x_off, y_traj - y_off],[x_traj[0] - x_off, y_traj + y_off]])

                theta = np.arctan2((y_o[indo][ind + 1] - y_o[indo][ind]) , (x_o[indo][ind + 1] - x_o[indo][ind])) - np.pi/2
                w = np.zeros(points.shape)
                A = np.matrix([[np.cos(theta), -np.sin(theta)],[np.sin(theta), np.cos(theta)]])
                points = points - np.array([x_traj[0], y_traj[0]])
                for i,v in enumerate(points): w[i] = A @ points[i]
                plt.fill(w[:, 0] + x_traj[0], w[:, 1] + y_traj[0], color=color['other'], zorder=4)

        # centerlines
        lane_centerlines = []
        # Get lane centerlines which lie within the range of trajectories
        agent_df = df[df['OBJECT_TYPE'] == 'AGENT']
        gt_x = agent_df['X'].values
        gt_y = agent_df['Y'].values

        x_max, y_max = np.max(x_a) + 50, np.max(y_a) + 50
        x_min, y_min = np.min(x_a) - 50, np.min(y_a) - 50

        # print(x_max, x_min)
        # print(y_max, y_min)
        for arr in avm.find_local_lane_polygons([x_min, x_max, y_min, y_max], city):
            plt.fill(arr[:, 0], arr[:, 1], color=color['polygon'],zorder=0)

        for arr in avm.find_local_lane_polygons([x_min, x_max, y_min, y_max], city):
            plt.plot(arr[:, 0], arr[:, 1], color=color['polygon-outline'],zorder=1)

        seq_lane_props = avm.city_lane_centerlines_dict[city]
        for lane_id, lane_props in seq_lane_props.items():
            lane_cl = lane_props.centerline
            if (np.min(lane_cl[:, 0]) < x_max and np.min(lane_cl[:, 1]) < y_max and np.max(lane_cl[:, 0]) > x_min and np.max(lane_cl[:, 1]) > y_min):
                lane_centerlines.append(lane_cl)

        for lane_cl in lane_centerlines:
            plt.plot(lane_cl[:, 0], lane_cl[:, 1], color=color['centerline'], alpha=1, linewidth=1, zorder=2)

#         plt.legend()
        plt.xlim([x_a[20] - 50, x_a[20] + 50])
        plt.ylim([y_a[20] - 50, y_a[20] + 50])
        import os
        try:
            os.mkdir('./results/{}'.format(idx))
        except:
            pass
#         plt.set_facecolor('red')
        plt.axis('off')
        plt.savefig('./results/{}/{}.png'.format(idx,ind), dpi=dpi, bbox_inches='tight')
        # fig.canvas.draw()
        # data_image = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
        # data_image = data_image.reshape(fig.canvas.get_width_height()[::-1] + (3,))
        # np.save('./results/{}/{}.npy'.format(idx,ind),  data_image)
        # print(data_image.shape)
        plt.clf()


from genericpath import samefile
import torch
# import mmcv
# from mmseg.apis import init_segmentor#, inference_segmentor, init_cfg
# from mmseg.models import build_segmentor
# from mmcv import ConfigDict

import torchvision

# from SETR.transformer_seg import SETRModel, Vit
import segmentation_models_pytorch as smp

def model_deeplab3(opt, singlecpop=False):
    model = torchvision.models.segmentation.deeplabv3_resnet50(
        # pretrained=True,
        progress=True,
        num_classes=opt.n_class if not singlecpop else 1
    )
    return model

def model_unet(opt, singlecpop=False):
    model = smp.Unet(
        encoder_name="resnet18",         # encoder
        encoder_depth=5,
        encoder_weights="imagenet",      # random initialization
        in_channels=len(opt.x_2D),                   # model input channels (1 for gray-scale images, 3 for RGB, etc.)
        classes=opt.n_class if not singlecpop else 1             # model output channels (number of classes in your dataset)
    )
    return model

def model_unet_(opt):
    cfg = mmcv.Config.fromfile('model/unet_cfg.py')
    # cfg = mmcv.Config.fromfile('/home/aya43/flowMagic_data/src/method/model/unet_cfg.py')

    model = init_segmentor(cfg)
    return model

def model_setr(opt, singlecpop=False):
    model = SETRModel(patch_size=(16, 16),
                    in_channels=len(opt.x_2D),
                    out_channels=opt.n_class if not singlecpop else 1,
                    hidden_size=1024,
                    num_hidden_layers=8,
                    num_attention_heads=8,
                    decode_features=[256, 128, 64, 32])
    # sum(p.numel() for p in model.parameters())
    t1 = torch.rand(1, 4, 256, 256)
    # print("input: " + str(t1.shape))
    print("output: " + str(model(t1).shape))
    return model

def model_setr_(opt):
    cfg = mmcv.Config.fromfile('model/vit_mla_cfg.py')
    # cfg = mmcv.Config.fromfile('/home/aya43/flowMagic_data/src/method/model/vit_mla_cfg.py')

    model = init_segmentor(cfg)
    return model

model_dict = {
    'setr': model_setr,
    'unet': model_unet,
    'deeplab3': model_deeplab3
}

model_names = list()
for name, dict_ in model_dict.items():
    model_names.append(name)

def create_model(opt, singlecpop):
    return model_dict[opt.model](opt, singlecpop)

def metafreeze_model(model, opt):
    # freeze
    for p in model.parameters():
        p.requires_grad = False

    if opt.model == 'setr':
        for p in model.encoder_2D.encoder.layer[5].parameters():
            p.requires_grad = True
        for p in model.encoder_2D.final_dense.parameters():
            p.requires_grad = True

    if opt.model == 'unet':
        for p in model.decoder.parameters():
            p.requires_grad = True

    return model