from tkinter import *
from tkinter.ttk import *
import psutil
import time
from threading import Thread
import sys

import utils as Utils
import const.color as Colors
import const.dimensions as Dimensions
import const.config as Config

window = Tk()
window.title(Config.APP_NAME_SHORT)
window.geometry(f"{Dimensions.SCREEN_WIDTH}x{Dimensions.SCREEN_HEIGHT}")
window.resizable(False, False)

icon = PhotoImage(file="img/icon.gif")
window.iconphoto(False, icon)

window.grid_columnconfigure(0, weight=1)

# <-- ROW 0 - START -->

row_0 = Frame(window)
row_0.grid(row=0, column=0, sticky="nsew")
row_0.grid_columnconfigure(0, weight=1)
row_0.grid_columnconfigure(1, weight=3)

# row 0 col 0 (General)
sys_info_frame = LabelFrame(row_0, text="General", padding=Dimensions.PADDING_L)
sys_info_frame.grid(row=0, column=0, sticky="nsew")

sys_info_frame.battery_label = Label(sys_info_frame)
sys_info_frame.battery_label.grid(row=0, sticky=W)
sys_info_frame.uptime_label = Label(sys_info_frame)
sys_info_frame.uptime_label.grid(row=1, sticky=W)
sys_info_frame.cpu_cores_label = Label(sys_info_frame)
sys_info_frame.cpu_cores_label.grid(row=2,sticky=W)

# row 0 col 1 (Memory)
memory_info_frame = LabelFrame(row_0, text="Memory", padding=Dimensions.PADDING_L)
memory_info_frame.grid(row=0, column=1, sticky="nsew")
memory_info_frame.grid_columnconfigure(0, weight=1)
memory_info_frame.grid_columnconfigure(1, weight=1)

memory_info_frame.mem_total = Label(memory_info_frame, font='Arial 12', foreground=Colors.GRAY)
memory_info_frame.mem_total.grid(row=0, column=0, sticky=W)
memory_info_frame.mem_used = Label(memory_info_frame, font='Arial 12', foreground=Colors.BLUE)
memory_info_frame.mem_used.grid(row=0, column=1, sticky=W)
memory_info_frame.mem_free = Label(memory_info_frame, font='Arial 12', foreground=Colors.BLACK)
memory_info_frame.mem_free.grid(row=0, column=2, sticky=W)
memory_info_frame.status_bar = Progressbar(memory_info_frame, orient=HORIZONTAL, mode="determinate", length=100)
memory_info_frame.status_bar.grid(row=1, columnspan=3, sticky=W+E)

# <-- ROW 0 - END -->

# <-- ROW 1 - START -->

row_1 = Frame(window)
row_1.grid(row=1, column=0, sticky="nsew")
row_1.grid_columnconfigure(0, weight=1)
row_1.grid_columnconfigure(1, weight=1)

# row 1 col 0 (RAM Usage)
ram_usage_frame = LabelFrame(row_1, text="RAM Usage", padding=Dimensions.PADDING_L)
ram_usage_frame.grid(row=0, column=0, sticky="nsew")
ram_usage_frame.grid_columnconfigure(0, weight=1)

ram_usage_frame.status_bar = Progressbar(ram_usage_frame, orient=HORIZONTAL, mode="determinate", length=100)
ram_usage_frame.status_bar.grid(row=0, sticky=W+E)
ram_usage_frame.status = Label(ram_usage_frame, font="Arial 15")
ram_usage_frame.status.grid(row=1, sticky=W+E)

# row 1 col 1 (CPU Usage)
cpu_usage_frame = LabelFrame(row_1, text="CPU Usage", padding=Dimensions.PADDING_L)
cpu_usage_frame.grid(row=0, column=1, sticky="nsew")
cpu_usage_frame.grid_columnconfigure(0, weight=1)

cpu_usage_frame.status_bar = Progressbar(cpu_usage_frame, orient=HORIZONTAL, mode="determinate", length=100)
cpu_usage_frame.status_bar.grid(row=0, sticky=W+E)
cpu_usage_frame.status = Label(cpu_usage_frame, font="Arial 15")
cpu_usage_frame.status.grid(row=1, sticky=W+E)

# <-- ROW 1 - END -->

# <-- ROW 2 - START -->

row_2 = Frame(window)
row_2.grid(row=2, column=0, sticky="nswe")
row_2.grid_columnconfigure(0, weight=1)

# row 2 col 0 (Tasks)
tasks_count_frame = LabelFrame(row_2, text="Tasks")
tasks_count_frame.grid(sticky="nswe")
tasks_count_frame.grid_columnconfigure(0, weight=1)
tasks_count_frame.grid_columnconfigure(1, weight=1)
tasks_count_frame.grid_columnconfigure(2, weight=1)
tasks_count_frame.grid_columnconfigure(3, weight=1)
tasks_count_frame.grid_columnconfigure(4, weight=1)

tasks_count_frame.total = Label(tasks_count_frame, font="Arial 12", padding=Dimensions.PADDING_S)
tasks_count_frame.total.grid(row=0, column=0, sticky="nsew")
tasks_count_frame.running = Label(tasks_count_frame, font="Arial 12", padding=Dimensions.PADDING_S)
tasks_count_frame.running.grid(row=0, column=1, sticky="nsew")
tasks_count_frame.sleeping = Label(tasks_count_frame, font="Arial 12", padding=Dimensions.PADDING_S)
tasks_count_frame.sleeping.grid(row=0, column=2, sticky="nsew")
tasks_count_frame.stopped = Label(tasks_count_frame, font="Arial 12", padding=Dimensions.PADDING_S)
tasks_count_frame.stopped.grid(row=0, column=3, sticky="nsew")
tasks_count_frame.zombie = Label(tasks_count_frame, font="Arial 12", padding=Dimensions.PADDING_S)
tasks_count_frame.zombie.grid(row=0, column=4, sticky="nsew")

# <-- ROW 2 - END -->

# <-- ROW 3 - START -->

row_3 = Frame(window)
row_3.grid(row=3, column=0, sticky="nswe")
row_3.grid_columnconfigure(0, weight=1)
row_3.grid_columnconfigure(1, weight=1)

# row 3 col 0 (Network)
network_frame = LabelFrame(row_3, text="Network", padding=Dimensions.PADDING_L)
network_frame.grid(row=0, column=0, sticky="nswe")
network_frame.grid_columnconfigure(0, weight=1)
network_frame.grid_columnconfigure(1, weight=1)
network_frame.grid_rowconfigure(0, weight=1)
network_frame.grid_rowconfigure(1, weight=1)

c00 = Frame(network_frame)
c00.grid(row=0, column=0, sticky="nswe")
c00.title = Label(c00, text="Download")
c00.title.grid(row=0, sticky=W)
c00.val = Label(c00, font='Arial 18')
c00.val.grid(row=1, sticky=W)

c01 = Frame(network_frame)
c01.grid(row=0, column=1, sticky="nswe")
c01.title = Label(c01, text="Download Speed")
c01.title.grid(row=0, sticky=W)
c01.val = Label(c01, foreground=Colors.GREEN, font='Arial 18')
c01.val.grid(row=1, sticky=W)

c10 = Frame(network_frame)
c10.grid(row=1, column=0, sticky="nswe")
c10.title = Label(c10, text="Upload")
c10.title.grid(row=0, sticky=W)
c10.val = Label(c10, font='Arial 18')
c10.val.grid(row=1, sticky=W)

c11 = Frame(network_frame)
c11.grid(row=1, column=1, sticky="nswe")
c11.title = Label(c11, text="Upload Speed")
c11.title.grid(row=0, sticky=W)
c11.val = Label(c11, foreground=Colors.RED, font='Arial 18')
c11.val.grid(row=1, sticky=W)

# row 3 col 1 (Top Precess)
top_process_frame = LabelFrame(row_3, text="Top process")
top_process_frame.grid(row=0, column=1, sticky="nswe")
top_process_frame.grid_columnconfigure(0, weight=1)
top_process_frame.grid_columnconfigure(1, weight=1)
top_process_frame.grid_columnconfigure(2, weight=1)
top_process_frame.grid_columnconfigure(3, weight=1)
top_process_frame.grid_columnconfigure(4, weight=1)

# table header starts
top_process_frame.c00 = Label(top_process_frame, text="pid", padding=Dimensions.PADDING_S, font='bold')
top_process_frame.c00.grid(row=0, column=0, sticky="w")
top_process_frame.c01 = Label(top_process_frame, text="Name", padding=Dimensions.PADDING_S, font='bold')
top_process_frame.c01.grid(row=0, column=1, sticky="w")
top_process_frame.c02 = Label(top_process_frame, text="Usage", padding=Dimensions.PADDING_S, font='bold')
top_process_frame.c02.grid(row=0, column=2, sticky="w")
top_process_frame.c03 = Label(top_process_frame, text="Status", padding=Dimensions.PADDING_S, font='bold')
top_process_frame.c03.grid(row=0, column=3, sticky="w")
top_process_frame.c04 = Label(top_process_frame, text="Threads", padding=Dimensions.PADDING_S, font='bold')
top_process_frame.c04.grid(row=0, column=4, sticky="w")
# table header ends

# table row 1 starts
top_process_frame.c10 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c10.grid(row=1, column=0, sticky="w")
top_process_frame.c11 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c11.grid(row=1, column=1, sticky="w")
top_process_frame.c12 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c12.grid(row=1, column=2, sticky="w")
top_process_frame.c13 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c13.grid(row=1, column=3, sticky="w")
top_process_frame.c14 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c14.grid(row=1, column=4, sticky="w")
# table row 1 ends

# table row 2 starts
top_process_frame.c20 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c20.grid(row=2, column=0, sticky="w")
top_process_frame.c21 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c21.grid(row=2, column=1, sticky="w")
top_process_frame.c22 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c22.grid(row=2, column=2, sticky="w")
top_process_frame.c23 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c23.grid(row=2, column=3, sticky="w")
top_process_frame.c24 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c24.grid(row=2, column=4, sticky="w")
# table row 2 ends

# table row 3 starts
top_process_frame.c30 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c30.grid(row=3, column=0, sticky="w")
top_process_frame.c31 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c31.grid(row=3, column=1, sticky="w")
top_process_frame.c32 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c32.grid(row=3, column=2, sticky="w")
top_process_frame.c33 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c33.grid(row=3, column=3, sticky="w")
top_process_frame.c34 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c34.grid(row=3, column=4, sticky="w")
# table row 3 ends

# table row 4 starts
top_process_frame.c40 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c40.grid(row=4, column=0, sticky="w")
top_process_frame.c41 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c41.grid(row=4, column=1, sticky="w")
top_process_frame.c42 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c42.grid(row=4, column=2, sticky="w")
top_process_frame.c43 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c43.grid(row=4, column=3, sticky="w")
top_process_frame.c44 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c44.grid(row=4, column=4, sticky="w")
# table row 4 ends

# table row 5 starts
top_process_frame.c50 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c50.grid(row=5, column=0, sticky="w")
top_process_frame.c51 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c51.grid(row=5, column=1, sticky="w")
top_process_frame.c52 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c52.grid(row=5, column=2, sticky="w")
top_process_frame.c53 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c53.grid(row=5, column=3, sticky="w")
top_process_frame.c54 = Label(top_process_frame, padding=Dimensions.PADDING_S)
top_process_frame.c54.grid(row=5, column=4, sticky="w")
# table row 5 ends

# <-- ROW 3 - END -->

# <-- ROW 4 - START -->

info_frame = Frame(window, padding=Dimensions.PADDING_L, relief=SUNKEN)
info_frame.grid(row=4, sticky="nswe")
info_frame.grid_columnconfigure(0, weight=1)

info_frame.title = Label(info_frame, text=Config.APP_NAME_LONG)
info_frame.title.grid(row=0)
auth_list = f"{Config.AUTH_AS} | {Config.AUTH_DF} | {Config.AUTH_SB} | {Config.AUTH_TP}"
info_frame.creators = Label(info_frame, text=auth_list)
info_frame.creators.grid(row=1)

# <-- ROW 4 - END -->

# NETWORK INFO CALCULATION - START
net_info_up = ""
net_info_dl = ""
net_info_up_speed = ""
net_info_dl_speed = ""

def network_info_stats():
    UPDATE_DELAY = 1
    io = psutil.net_io_counters()
    bytes_sent, bytes_recv = io.bytes_sent, io.bytes_recv
    while True:
        time.sleep(UPDATE_DELAY)
        io_2 = psutil.net_io_counters()
        us, ds = io_2.bytes_sent - bytes_sent, io_2.bytes_recv - bytes_recv
        global net_info_up
        net_info_up = Utils.get_size(io_2.bytes_sent)
        global net_info_dl
        net_info_dl = Utils.get_size(io_2.bytes_recv)
        global net_info_up_speed
        net_info_up_speed = Utils.get_size(us / UPDATE_DELAY)
        global net_info_dl_speed
        net_info_dl_speed = Utils.get_size(ds / UPDATE_DELAY)
        bytes_sent, bytes_recv = io_2.bytes_sent, io_2.bytes_recv

# a thread to find the network information
network_info_thread = Thread(target=network_info_stats)
network_info_thread.start()
# NETWORK INFO CALCULATION - START

app_running = True

while app_running:

    battery_percentage = psutil.sensors_battery().percent
    sys_info_frame.battery_label.config(text=f"Battery: {battery_percentage:.2f}%")

    up_time = time.time() - psutil.boot_time()
    up_time_hrs = up_time / 60 / 60
    final_up_time = ''
    if up_time_hrs > 1:
        final_up_time = str(f"{up_time_hrs:.2f} hrs")
    else:
        final_up_time = str(f"{up_time/60:.2f} min")
    sys_info_frame.uptime_label.config(text=f"Up Time: {final_up_time}")

    sys_info_frame.cpu_cores_label.config(text=f"CPU cores: {psutil.cpu_count()}")

    disk_usage = psutil.disk_usage('/')
    percentage_used = round(disk_usage.percent)
    total_memory = str(round(disk_usage.total / (1024.0 ** 3))) + 'GB'
    free_memory = str(round(disk_usage.free / (1024.0 ** 3))) + 'GB'
    used_memory = str(round(disk_usage.used / (1024.0 ** 3))) + 'GB'

    memory_info_frame.mem_total.config(text=f"Total: {total_memory}")
    memory_info_frame.mem_used.config(text=f"Used: {used_memory}")
    memory_info_frame.mem_free.config(text=f"Free: {free_memory}")

    disk_usage = psutil.disk_usage('/')
    percentage_used = round(disk_usage.percent)
    memory_info_frame.status_bar.config(value=percentage_used)

    ram_percentage = psutil.virtual_memory().percent
    percentage_used = round(ram_percentage)
    ram_usage_frame.status_bar.config(value=percentage_used)
    ram_usage_frame.status.config(text=f"{ram_percentage}%")

    cpu_percentage = psutil.cpu_percent()
    percentage_used = round(cpu_percentage)
    cpu_usage_frame.status_bar.config(value=percentage_used)
    cpu_usage_frame.status.config(text=f"{cpu_percentage}%")

    total_proc_no = 0
    running_proc_no = 0
    sleeping_proc_no = 0
    zombie_proc_no = 0
    stopped_proc_no = 0
    process_list = []
    for proc in psutil.process_iter():
        mem_p = round(proc.memory_percent(),2)
        process_list.append([
            proc.pid,
            proc.name(),
            f"{mem_p}%",
            proc.status(),
            proc.num_threads()
        ])
        total_proc_no += 1
        if (proc.status() == psutil.STATUS_RUNNING):
            running_proc_no += 1
        elif (proc.status() == psutil.STATUS_SLEEPING):
            sleeping_proc_no += 1
        elif (proc.status() == psutil.STATUS_ZOMBIE):
            zombie_proc_no += 1
        elif proc.status() == psutil.STATUS_STOPPED:
            stopped_proc_no += 1

    process_list.sort(key=lambda x:x[2], reverse=True)

    tasks_count_frame.total.config(text=f"Total:{total_proc_no}")
    tasks_count_frame.running.config(text=f"Running:{running_proc_no}")
    tasks_count_frame.sleeping.config(text=f"Sleeping:{sleeping_proc_no}")
    tasks_count_frame.stopped.config(text=f"Stopped:{stopped_proc_no}")
    tasks_count_frame.zombie.config(text=f"Zombie:{zombie_proc_no}")

    c00.val.config(text=net_info_up)
    c01.val.config(text=f"{net_info_up_speed}/s")
    c10.val.config(text=net_info_dl)
    c11.val.config(text=f"{net_info_dl_speed}/s")

    top_process_frame.c10.config(text=process_list[0][0])
    top_process_frame.c11.config(text=process_list[0][1])
    top_process_frame.c12.config(text=process_list[0][2])
    top_process_frame.c13.config(text=process_list[0][3])
    top_process_frame.c14.config(text=process_list[0][4])

    top_process_frame.c20.config(text=process_list[1][0])
    top_process_frame.c21.config(text=process_list[1][1])
    top_process_frame.c22.config(text=process_list[1][2])
    top_process_frame.c23.config(text=process_list[1][3])
    top_process_frame.c24.config(text=process_list[1][4])

    top_process_frame.c30.config(text=process_list[2][0])
    top_process_frame.c31.config(text=process_list[2][1])
    top_process_frame.c32.config(text=process_list[2][2])
    top_process_frame.c33.config(text=process_list[2][3])
    top_process_frame.c34.config(text=process_list[2][4])

    top_process_frame.c40.config(text=process_list[3][0])
    top_process_frame.c41.config(text=process_list[3][1])
    top_process_frame.c42.config(text=process_list[3][2])
    top_process_frame.c43.config(text=process_list[3][3])
    top_process_frame.c44.config(text=process_list[3][4])

    top_process_frame.c50.config(text=process_list[4][0])
    top_process_frame.c51.config(text=process_list[4][1])
    top_process_frame.c52.config(text=process_list[4][2])
    top_process_frame.c53.config(text=process_list[4][3])
    top_process_frame.c54.config(text=process_list[4][4])

    time.sleep(0.1)
    window.update()

window.mainloop()

def disable_event():
    network_info_thread.join()
    window.quit()
    window.destroy()
    sys.exit()

window.protocol("WM_DELETE_WINDOW", disable_event)


from bienal import BienalClient, get_deep_ai_caption_position
from random import choice

img = None
analysis = None

def settings():
    global img, analysis

    bienal = BienalClient()
    print("### GET IMAGE")
    analysis = bienal.get_collection_image(col_id=16, image_id=942)

    url = analysis['image']
    img = loadImage(url)
    img.resize(900, 0)
    print(img.width, img.height)
    size(img.width, img.height)
    print('DONE')

def setup():
    pass

def draw():
    properties = analysis.google.image_properties_annotation

    colors = [c['color'] for c in properties['dominantColors']['colors']]
    img.loadPixels()

    for y in range(height):
        max_x = int(noise(y / 89.0) * width)
        if random(1) > 0.83:
            continue

        for x in range(max_x):
            index = x + y * width
            c = choice(colors)
            img.pixels[index] = color(c['red'], c['green'], c['blue'])

    img.updatePixels()
    image(img, 0, 0)
    saveFrame("img_001.png")
    noLoop()


import bibtexparser
import pandas as pd
import subprocess
from subprocess import check_output
import os
import re
import nltk
import doi2pdf as d2p
import textanalysis as ta

def bib2csv(entries):
	df = pd.DataFrame(entries)
	df.to_csv("merged.csv", index=False)
	return

def temporalOverview(df):
	mostrecent = pd.to_numeric(df["year"]).max()
	earliest = pd.to_numeric(df["year"]).min()
	print("... spanning time range of", mostrecent-earliest,"years: from",earliest, "to",mostrecent)

	perYear = df["year"].value_counts()
	print("\nranked by number of entries:\n")
	print(perYear)

	print("\nranked by year:\n")
	print(perYear.sort_index())

	return

def conferenceOverview(df):
	series = df["series"].value_counts()
	print("\nranked by conference:\n")
	print(series)

	return

# create new column of overarching conference (i.e., group all CHIs regardless of year) - works for ACM DL where series = <NAME> <YEAR AS 'XX> <OPTIONAL REST OF NAME>
def confHat(df):
	df["confName"] = df["series"].apply(lambda row: splitName(row))
	#print(df["confName"])
	confNames = df["confName"].value_counts()
	print("\nranked by conference whole:\n")
	print(confNames)

	return

def splitName(name):
	if (pd.isna(name)):
		return "--"
	# split by: optional "´" char, then 2-4 digits:
	confNoYear =  re.split("’?\d{2,4}", name)
	confNoYear =  "".join(confNoYear)
	return confNoYear

if __name__== "__main__":

	# make sure a filepath argument has been added:
	#if (len(sys.argv) != 2):
	#	print("... need a filepath appended to this command, to indicate a folder containing *.bib files.")
	#	quit()

	# firstarg = sys.argv[1]

	path = subprocess.Popen("pwd", stdout=subprocess.PIPE) # returns output and return code
	path = path.communicate()[0]
	path = path.decode("utf-8")
	print("... searching for bibs in: \n\n", path, "\n")

	# gather bibfiles:
	countbibs = 0
	bibFiles = []
	with os.scandir() as i:
		for entry in i:
			if entry.is_file():
				fileEnding = entry.name.split(".")
				fileEnding = fileEnding[len(fileEnding)-1]

				if (fileEnding == "bib"):
					countbibs += 1
					bibFiles.append(entry.name)
	print("... found ",countbibs, "bib files:")
	print(bibFiles, "\n")

	if (bibFiles == 0):
		print("...quitting.")
		quit()

	# set parser to recognize common strings like "nov"
	parser = bibtexparser.bparser.BibTexParser(common_strings=True)

	# merge bibfiles
	for bib in bibFiles:
		with open(bib, "r") as bpfile:
			bpdb = bibtexparser.load(bpfile, parser = parser)
			# print(len(bpdb.entries))
	print("... with a total of ", len(bpdb.entries), "bib entries.")

	# save as merged csv:
	bib2csv(bpdb.entries)

	pandaDF = pd.DataFrame(bpdb.entries)
	print(pandaDF.columns.values)

	# temporal overview
	temporalOverview(pandaDF)

	# conference overview - by conference series vs. grouped by conference regardless of year
	conferenceOverview(pandaDF)
	confHat(pandaDF)

	print(pandaDF[["confName","booktitle","doi"]])

	# pandaDF.ID! doi not always entered
	#d2p.doi2pdf("10.1145/3173574.3173660")

	d2p.scrapeFromDOIs(pandaDF["ID"])

	# turn PDFs into text
	#ta.pdf2text("pdfs/" + "10.1145/3173574.3173660".replace("/","--") + ".pdf")
	ta.extractText()

	# test nltk: via python ./textanalysis.py


import _pickle as cPickle
import numpy as np

INPUTS = 1836
TAGS = 159

# Loading all the datasets data
dataset_names = ['train', 'test']
datasets = {name: [] for name in dataset_names}
for d in dataset_names:
    with open('data/bibtex/bibtex-%s.arff' % d, 'r') as f:
        data_started = False
        for line in f:
            if line.strip() == "@data":
                data_started = True
                continue
            if data_started is False:
                continue
            line = line.replace("{", "")
            line = line.replace("}", "")
            tokens = [int(x.split()[0]) for x in line.split(",")]
            datasets[d].append({
                'sparse_feats': [t for t in tokens if t < INPUTS],
                'sparse_types': [t - INPUTS for t in tokens if t >= INPUTS]
            })

# Copying test set to form dev set
datasets['dev'] = datasets['test']

for name, dataset in datasets.items():
    for instance in dataset:
        feats_vector = np.zeros(INPUTS)
        feats_vector[instance['sparse_feats']] = 1
        types_vector = np.zeros(TAGS)
        types_vector[instance['sparse_types']] = 1
        instance['feats'] = feats_vector
        instance['types'] = types_vector
    with open('data/bibtex/%s.pickle' % name, 'wb') as f:
        cPickle.dump(dataset, f)


import json
import os.path

import geopy.distance
import networkx as nx
import numpy as np
import osmnx as ox
from geojson import FeatureCollection
from osgeo import ogr
from shapely.geometry import MultiPoint, Polygon
from tqdm import tqdm

from tracking_decorator import TrackingDecorator

def get_isochrone(logger, graph, start_point, travel_time, distance_attribute="time"):
    try:
        return get_possible_routes(
            graph=graph,
            start_point=start_point,
            travel_time=travel_time,
            distance_attribute=distance_attribute,
            calculate_walking_distance=True
        )
    except Exception as e:
        logger.log_line(f"✗ Exception: {str(e)}")
        return None, 0

def get_isochrone_metrics(logger, data_path, city_id, start_point, subgraph, walking_distance_meters):
    try:
        nodes, edges = ox.graph_to_gdfs(subgraph)

        # Define valid polygons
        polygon_file = os.path.join(data_path, "cities", city_id, "boundaries", "boundaries.geojson")
        valid_polygons = get_polygons(read_geojson(polygon_file))

        longitudes, latitudes = get_convex_hull(nodes=nodes)
        points = []

        # Filter points that are outside city limits
        for point in zip(longitudes, latitudes):

            p = ogr.Geometry(ogr.wkbPoint)
            p.AddPoint(point[0], point[1])

            # Check if point is within city limits
            if is_in_desired_area(p, valid_polygons):
                points.append([point[0], point[1]])
            else:
                logger.log_line(f"{city_id} point outside city {point[0]}, {point[1]}")

        convex_hull_polygon = Polygon(points)

        transport_distances_meters = get_distances(start_point=start_point, points=points)

        return np.mean(transport_distances_meters) + walking_distance_meters, \
               np.median(transport_distances_meters) + walking_distance_meters, \
               np.min(transport_distances_meters) + walking_distance_meters, \
               np.max(transport_distances_meters) + walking_distance_meters, \
               convex_hull_polygon.area
    except Exception as e:
        logger.log_line(f"✗ Exception: {str(e)}")
        return 0, 0, 0, 0, 0

def is_in_desired_area(point, valid_polygons):
    for polygon in valid_polygons:
        if point.Within(polygon):
            return True

    return False

def read_geojson(file_path):
    file = open(file_path)
    return json.load(file)

def get_polygons(geojson):
    polygons = []

    # Extract polygons from inhabitants file
    features = geojson["features"]

    for feature in features:
        geom = feature["geometry"]
        geom = json.dumps(geom)
        polygon = ogr.CreateGeometryFromJson(geom)

        polygons.append(polygon)

    return polygons

def get_possible_routes(graph, start_point, travel_time, distance_attribute, calculate_walking_distance=False):
    center_node, distance_to_nearest_node = ox.nearest_nodes(
        G=graph,
        X=start_point[1],
        Y=start_point[0],
        return_dist=True
    )

    if calculate_walking_distance:
        walking_speed_meters_per_minute = 100
        walking_time_minutes = distance_to_nearest_node / walking_speed_meters_per_minute

        walking_time_minutes_max = walking_time_minutes if walking_time_minutes < travel_time else travel_time
        walking_distance_meters = walking_time_minutes_max * walking_speed_meters_per_minute

        radius = travel_time - walking_time_minutes
    else:
        walking_distance_meters = 0
        radius = travel_time

    if radius > 0:
        return nx.ego_graph(graph, center_node, radius=radius, distance=distance_attribute), walking_distance_meters
    else:
        return None, walking_distance_meters

def get_convex_hull(nodes):
    return MultiPoint(nodes.reset_index()["geometry"]).convex_hull.exterior.coords.xy

def get_distances(start_point, points):
    return [geopy.distance.geodesic(point, start_point).meters for point in points]

def write_points_to_geojson(file_path, coords, travel_time):
    features = []
    for coord in coords:
        feature = {}
        feature["geometry"] = {"type": "Point", "coordinates": [coord["lon"], coord["lat"]]}
        feature["type"] = "Feature"
        feature["properties"] = {
            "mean_spatial_distance_" + str(travel_time) + "min": coord[
                "mean_spatial_distance_" + str(travel_time) + "min"],
            "median_spatial_distance_" + str(travel_time) + "min": coord[
                "median_spatial_distance_" + str(travel_time) + "min"],
            "min_spatial_distance_" + str(travel_time) + "min": coord[
                "min_spatial_distance_" + str(travel_time) + "min"],
            "max_spatial_distance_" + str(travel_time) + "min": coord[
                "max_spatial_distance_" + str(travel_time) + "min"],
            "area_" + str(travel_time) + "min": coord[
                "area_" + str(travel_time) + "min"],
        }
        features.append(feature)

    collection = FeatureCollection(features)

    with open(file_path, "w") as f:
        f.write("%s" % collection)

def write_nodes_to_geojson(file_path, graph):
    features = []

    if len(graph.nodes) > 0:
        for node_id in graph.nodes:
            node = graph.nodes[node_id]
            feature = {}
            feature["geometry"] = {"type": "Point", "coordinates": [node["x"], node["y"]]}
            feature["type"] = "Feature"
            features.append(feature)

    collection = FeatureCollection(features)

    with open(file_path, "w") as f:
        f.write("%s" % collection)

def write_polygon_to_geojson(file_path, graph):
    features = []
    coordinates = []

    for node_id in graph.nodes:
        node = graph.nodes[node_id]
        coordinates.append([node["x"], node["y"]])

    if len(graph.nodes) > 0:
        feature = {}
        feature["geometry"] = {"type": "Polygon", "coordinates": coordinates}
        feature["type"] = "Feature"
        features.append(feature)

    collection = FeatureCollection(features)

    with open(file_path, "w") as f:
        f.write("%s" % collection)

#
# Main
#

class IsochroneBuilder:

    @TrackingDecorator.track_time
    def run(self, logger, data_path, results_path, city_id, graph, sample_points, travel_time, start_time, end_time):
        points_with_spatial_distance = []
        failed_points = []

        # Make results path
        os.makedirs(os.path.join(results_path), exist_ok=True)

        for point_index in tqdm(iterable=range(len(sample_points)),
                                total=len(sample_points),
                                desc="Build isochrone",
                                unit="point"):
            point = sample_points[point_index]
            start_point = (float(point["lat"]), float(point["lon"]))

            # Calculate isochrone
            subgraph, walking_distance_meters = get_isochrone(
                logger=logger,
                graph=graph,
                start_point=start_point,
                travel_time=travel_time
            )

            # Determine isochrone metrics
            mean_spatial_distance, median_spatial_distance, min_spatial_distance, max_spatial_distance, area = get_isochrone_metrics(
                logger=logger,
                data_path=data_path,
                city_id=city_id,
                start_point=start_point,
                subgraph=subgraph,
                walking_distance_meters=walking_distance_meters
            )

            point_with_spatial_distance = {
                "lon": point["lon"],
                "lat": point["lat"],
                "mean_spatial_distance_" + str(travel_time) + "min": mean_spatial_distance,
                "median_spatial_distance_" + str(travel_time) + "min": median_spatial_distance,
                "min_spatial_distance_" + str(travel_time) + "min": min_spatial_distance,
                "max_spatial_distance_" + str(travel_time) + "min": max_spatial_distance,
                "area_" + str(travel_time) + "min": area
            }

            if mean_spatial_distance > 0:
                points_with_spatial_distance.append(point_with_spatial_distance)
            else:
                failed_points.append(point_with_spatial_distance)

        write_points_to_geojson(
            file_path=os.path.join(results_path, f"isochrones-{str(travel_time)}min-{start_time}-{end_time}.geojson"),
            coords=points_with_spatial_distance,
            travel_time=travel_time
        )

        write_points_to_geojson(
            file_path=os.path.join(results_path, f"isochrones-{str(travel_time)}min-{start_time}-{end_time}-failed.geojson"),
            coords=failed_points,
            travel_time=travel_time
        )

        return points_with_spatial_distance, failed_points

    @TrackingDecorator.track_time
    def run_for_place(self, logger, data_path, results_path, city_id, graph, travel_time, place):
        points_with_spatial_distance = []
        failed_points = []
        start_point = (place[1], place[0])

        # Make results path
        os.makedirs(os.path.join(results_path), exist_ok=True)

        # Calculate isochrone
        subgraph, walking_distance_meters = get_isochrone(
            logger=logger,
            graph=graph,
            start_point=start_point,
            travel_time=travel_time
        )

        # Determine isochrone metrics
        mean_spatial_distance, median_spatial_distance, min_spatial_distance, max_spatial_distance, area = get_isochrone_metrics(
            logger=logger,
            data_path=data_path,
            city_id=city_id,
            start_point=start_point,
            subgraph=subgraph,
            walking_distance_meters=walking_distance_meters
        )

        write_nodes_to_geojson(file_path=os.path.join(results_path, "isochrone-nodes-" + str(travel_time) + ".geojson"), graph=subgraph)
        write_polygon_to_geojson(file_path=os.path.join(results_path, "isochrone-hull-" + str(travel_time) + ".geojson"), graph=subgraph)

        point_with_spatial_distance = {
            "lon": start_point[1],
            "lat": start_point[0],
            "mean_spatial_distance_" + str(travel_time) + "min": mean_spatial_distance,
            "median_spatial_distance_" + str(travel_time) + "min": median_spatial_distance,
            "min_spatial_distance_" + str(travel_time) + "min": min_spatial_distance,
            "max_spatial_distance_" + str(travel_time) + "min": max_spatial_distance,
            "area_" + str(travel_time) + "min": area
        }

        if mean_spatial_distance > 0:
            points_with_spatial_distance.append(point_with_spatial_distance)
        else:
            failed_points.append(point_with_spatial_distance)

        write_points_to_geojson(
            file_path=os.path.join(results_path, "isochrones-" + str(travel_time) + ".geojson"),
            coords=points_with_spatial_distance,
            travel_time=travel_time
        )

        write_points_to_geojson(
            file_path=os.path.join(results_path, "isochrones-" + str(travel_time)) + "-failed.geojson",
            coords=failed_points,
            travel_time=travel_time
        )

        return points_with_spatial_distance, failed_points