## COEUS: app to analyze trading performance

## IMPORTS
import streamlit as st
import altair as alt
import pandas as pd
import yfinance as yf
import numpy as np
from scipy import stats
from datetime import timedelta

# WEBSITE

# Intro layout
st.title('**Coeus - Trading Performance Analyzer**')
st.write('---')

st.header('*INTRODUCTION*')
st.subheader('Why Coeus?')
st.markdown('Coeus is the Greek Titan of Intelligence, we hope that by analyzing and gaining insights into your portfolio, you may gain some of insight into your trading performance')
st.subheader('What is Coeus?')
st.markdown('Coeus is an app which quantifies your trading performance, and allows you to gain insights into how well you trade, and finds ways you can optimize your portfolio')
st.subheader('How does it work?')
st.markdown('By analyzing your trading data, we can quantify the levels of risk, the returns, the level of diversity and much more about your portfolio. This allows us to combine all this data into a single number, the Indicator. You can then observe this value over time to see if your trades are improving ')
st.write('---')

# Input layout
st.header('*INPUT*')
st.subheader('How to input your trading data')
st.markdown('Step 1: Fetch your trading data from your broker. This must be in an excel sheet form')
st.markdown('Step 2: Convert your data by following the example below. Be careful not to include the currency in the excel sheet, only the numerical values. And the date must be in UK format: dd/mm/yyyy. And the symbol must be all caps')
example_input = pd.DataFrame(data={'Instrument name': ['Tesla Inc.'], 'Instrument symbol': ['TSLA'], 'Buy price': ['109.32'], 'Buy date': ['13/03/2020'], 'Sell price': ['274.88'], 'Sell date': ['11/08/2020'], 'Quantity': ['100']})
st.dataframe(example_input)
st.markdown('Step 3: Transform your excel sheet into the CSV format, and upload it using the button below')
st.markdown('Example of CSV input')
uploaded_file = st.file_uploader("Upload CSV", type=['csv'])
st.write('---')

# Overview layout
if uploaded_file is not None:
    df = pd.read_csv(uploaded_file)

    ## CALCULATIONS
    # convert the 'Date' column to datetime format and finding min and max dates
    df['Buy date'] = pd.to_datetime(df['Buy date'])
    df['Sell date'] = pd.to_datetime(df['Sell date'])
    portfolio_start_date = df['Buy date'].min()
    portfolio_end_date = df['Sell date'].max()

    # Portfolio value
    portfolio_value = pd.Series(index=['Close'])
    for i in range(0,len(df.index)):
        ticker = df['Instrument symbol'][i]
        ticker_start_date = df['Buy date'][i]
        ticker_end_date = df['Sell date'][i]
        tickerDf = yf.Ticker(ticker)
        tickerData = tickerDf.history(start=ticker_start_date, end=ticker_end_date)
        tickerPrices = tickerData['Close']
        ## error management here, if doesnt find stock from yfinance
        position_value = tickerPrices*df['Quantity'][i]
        portfolio_value = portfolio_value.add(position_value, fill_value=0)
    date_range = pd.date_range(start=portfolio_start_date, end=portfolio_end_date)
    portfolio_value = pd.DataFrame(portfolio_value, index=date_range)
    portfolio_value = portfolio_value.dropna()

    # Overall portfolio performance metrics
    total_sell = sum(df['Sell price']*df['Quantity'])
    total_buy = sum(df['Buy price']*df['Quantity'])
    profit_loss = total_sell - total_buy

    total_returns = ((total_sell - total_buy)/total_buy)*100

    #benchmark = yf.Ticker('^GSPC')
    #benchmark_values = benchmark.history(start=portfolio_start_date, end=portfolio_end_date)
    #benchmark_ret1 = benchmark_values['Close'].pct_change()
    #benchmark_ret = benchmark_ret1.to_numpy()
    #st.write(benchmark_ret)
    #port_ret1 = portfolio_value.pct_change()
    #port_ret = port_ret1.to_numpy()
    #st.write(port_ret)
    #covariance = np.cov(port_ret, benchmark_ret)
    #portfolio_beta = covariance[0, 1]/covariance[1, 1]
    #treynor_measure = total_returns / portfolio_beta

    #portfolio_standard_deviation = portfolio_value.std()
    #sharpe_ratio = total_returns / portfolio_standard_deviation

    #beta_jensen =
    #jensen_measure = (total_returns - risk_free_rate - beta_jensen)*100

    # Asset returns for bar chart
    d = list()
    for i in range(0, len(df.index)):
        ticker = df['Instrument symbol'][i]
        tickerReturns = ((df['Sell price'][i]-df['Buy price'][i])/df['Buy price'][i])*100
        d.append(tickerReturns)
    asset_returns = pd.DataFrame(data=d, index=df['Instrument name'])

    # Analysis layout OVERVIEW
    st.header('*ANALYSIS*')
    st.subheader('Overview')

    st.markdown('List of trades:')
    st.dataframe(df)

    st.markdown('Portfolio valuation over time:')
    st.area_chart(portfolio_value)
    # make a chart with layers for each stock, use altair

    st.markdown('Performance metrics:')
    st.text('Profit-Loss: %s dollars' % int(profit_loss))
    st.text('Total rate of return: %s percent' % round(total_returns,1))
    #st.text('Treynor measure: %s' % treynor_measure)
    #st.text('Sharpe ratio: %s' % sharpe_ratio)
    #st.text('Jensen measure: %s percent' % jensen_measure)

    st.markdown('Asset returns:')
    st.bar_chart(asset_returns)

    # Analysis layout ASSETS
    st.subheader('Asset analysis')
    option2 = st.selectbox('Asset', df['Instrument name'])
    row_index = df[df['Instrument name'] == option2].index
    instrument_symbol = df['Instrument symbol'][row_index].to_list()
    start_date2 = df['Buy date'][row_index].to_list()
    end_date2 = df['Sell date'][row_index].to_list()
    tickerData2 = yf.Ticker(instrument_symbol[0])  # Get ticker data
    tickerDf2 = tickerData2.history(start=start_date2[0], end=end_date2[0])  # Get the historical prices for the ticker

    st.markdown('Information:')
    string_summary = tickerData2.info['longBusinessSummary']
    st.markdown('%s' % string_summary)

    st.markdown('Return:')

    st.markdown('Benchmark S&P 500 comparison:')
    benchmarkData = yf.Ticker('^GSPC')
    benchmarkDf = benchmarkData.history(start=start_date2[0], end=end_date2[0])

    d = {'ticker': ((tickerDf2['Close'] - tickerDf2['Close'][0]) / tickerDf2['Close'][0]) * 100,
     'benchmark': ((benchmarkDf['Close'] - benchmarkDf['Close'][0]) / benchmarkDf['Close'][0]) * 100}
    benching = pd.DataFrame(data=d)

    tickerReturns = ((tickerDf2['Close'][-1] - tickerDf2['Close'][0]) / tickerDf2['Close'][0]) * 100
    tickerReturns = round(tickerReturns, 3)
    benchmarkReturns = ((benchmarkDf['Close'][-1] - benchmarkDf['Close'][0]) / benchmarkDf['Close'][0]) * 100
    benchmarkReturns = round(benchmarkReturns, 3)
    st.text('The stock you choose made %s percent returns over the time period' % tickerReturns)
    st.text('The benchmark (S&P500) made %s percent returns over the time period' % benchmarkReturns)

    # Plotting the data
    st.line_chart(benching)


#!/usr/bin/env python

import logging

LOG_LEVEL = logging.INFO
ZIP_OUTPUT = True

repo_name = 'npg'
repo_path = './../../aac-repos/' + repo_name
base_uri = 'http://data.americanartcollaborative.org/npg/'
context_uri = 'https://github.com/american-art/aac-alignment/blob/master/karma-context.json'
REPO_CONFIG = [
    {
        'path': repo_path,
        'name': 'NPGBibReferences',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E31_Document1',
        'context_uri': context_uri,
        'model_file': 'npgbibreferences-model.ttl',
        'input_file': 'npgbibreferences.csv',
        'output_file_name': 'npgbibreferences'
    },
    {
        'path': repo_path,
        'name': 'NPGConAltNames',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E39_Actor1',
        'context_uri': context_uri,
        'model_file': 'NPGConAltNames2-Institutions-model.ttl',
        'input_file': 'NPGConAltNames2.csv',
        'output_file_name': 'NPGConAltNames2-Institutions'
    },
    {
        'path': repo_path,
        'name': 'NPGConAltNames',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E39_Actor1',
        'context_uri': context_uri,
        'model_file': 'NPGConAltNames2-People-model.ttl', # different model
        'input_file': 'NPGConAltNames2.csv',
        'output_file_name': 'NPGConAltNames2-People'
    },
    {
        'path': repo_path,
        'name': 'NPGConstituents',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E39_Actor1',
        'context_uri': context_uri,
        'model_file': 'NPGConstituents2-People-model.ttl',
        'input_file': 'NPGConstituents2.csv',
        'output_file_name': 'NPGConstituents2-People'
    },
    {
        'path': repo_path,
        'name': 'NPGConstituents',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E39_Actor1',
        'context_uri': context_uri,
        'model_file': 'NPGConstituents2-Institutions-model.ttl',
        'input_file': 'NPGConstituents2.csv',
        'output_file_name': 'NPGConstituents2-Institutions'
    },
    {
        'path': repo_path,
        'name': 'NPGConThesTerms',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E39_Actor1',
        'context_uri': context_uri,
        'model_file': 'NPGConThesTerms2-model.ttl',
        'input_file': 'NPGConThesTerms2.csv',
        'output_file_name': 'NPGConThesTerms2'
    },
    {
        'path': repo_path,
        'name': 'NPGDimsParsedUpdate2May',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E22_Man-Made_Object1',
        'context_uri': context_uri,
        'model_file': 'NPGDimsParsedUpdate2May-model.ttl',
        'input_file': 'NPGDimsParsedUpdate2May.csv',
        'output_file_name': 'NPGDimsParsedUpdate2May'
    },
    {
        'path': repo_path,
        'name': 'NPGExhibitionObjXrefs',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/PC16_used_specific_object1',
        'context_uri': context_uri,
        'model_file': 'NPGExhibitionObjXrefs2-model.ttl',
        'input_file': 'NPGExhibitionObjXrefs2.csv',
        'output_file_name': 'NPGExhibitionObjXrefs2'
    },
    {
        'path': repo_path,
        'name': 'NPGExhibitions',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/PC16_used_specific_object1',
        'context_uri': context_uri,
        'model_file': 'NPGExhibitions2-model.ttl',
        'input_file': 'NPGExhibitions2.csv',
        'output_file_name': 'NPGExhibitions2'
    },
    {
        'path': repo_path,
        'name': 'NPGObjConXrefs',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E22_Man-Made_Object1',
        'context_uri': context_uri,
        'model_file': 'NPGObjConXrefs2-model.ttl',
        'input_file': 'NPGObjConXrefs2.csv',
        'output_file_name': 'NPGObjConXrefs2'
    },
    {
        'path': repo_path,
        'name': 'NPGObjects',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E22_Man-Made_Object1',
        'context_uri': context_uri,
        'model_file': 'NPGObjects2-model.ttl',
        'input_file': 'NPGObjects2.csv',
        'output_file_name': 'NPGObjects2'
    },
    {
        'path': repo_path,
        'name': 'NPGObjExhText',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E22_Man-Made_Object1',
        'context_uri': context_uri,
        'model_file': 'NPGObjExhText2-model.ttl',
        'input_file': 'NPGObjExhText2.csv',
        'output_file_name': 'NPGObjExhText2'
    },
    {
        'path': repo_path,
        'name': 'NPGObjProvenance',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E22_Man-Made_Object1',
        'context_uri': context_uri,
        'model_file': 'NPGObjProvenance_2-model.ttl',
        'input_file': 'NPGObjProvenance_2.csv',
        'output_file_name': 'NPGObjProvenance_2'
    },
    {
        'path': repo_path,
        'name': 'NPGObjThesTerms',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E22_Man-Made_Object1',
        'context_uri': context_uri,
        'model_file': 'NPGObjThesTerms2-model.ttl',
        'input_file': 'NPGObjThesTerms2.csv',
        'output_file_name': 'NPGObjThesTerms2'
    },
    {
        'path': repo_path,
        'name': 'NPGObjTitles',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E22_Man-Made_Object1',
        'context_uri': context_uri,
        'model_file': 'NPGObjTitles2-model.ttl',
        'input_file': 'NPGObjTitles2.csv',
        'output_file_name': 'NPGObjTitles2'
    },
    {
        'path': repo_path,
        'name': 'NPGObjURLs',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E22_Man-Made_Object1',
        'context_uri': context_uri,
        'model_file': 'NPGObjURLs-model.ttl',
        'input_file': 'NPGObjURLs.csv',
        'output_file_name': 'NPGObjURLs'
    },
    {
        'path': repo_path,
        'name': 'NPGObjWebLabels',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E22_Man-Made_Object1',
        'context_uri': context_uri,
        'model_file': 'NPGObjWebLabels2-model.ttl',
        'input_file': 'NPGObjWebLabels2.csv',
        'output_file_name': 'NPGObjWebLabels2'
    },
    {
        'path': repo_path,
        'name': 'NPGRefObjXrefs',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E31_Document1',
        'context_uri': context_uri,
        'model_file': 'NPGRefObjXrefs-model.ttl',
        'input_file': 'NPGRefObjXrefs.csv',
        'output_file_name': 'NPGRefObjXrefs'
    },
    {
        'path': repo_path,
        'name': 'NPGWebImageURLs',
        'base_uri': base_uri,
        'rdf_root_uri': 'http://www.cidoc-crm.org/cidoc-crm/E22_Man-Made_Object1',
        'context_uri': context_uri,
        'model_file': 'NPGWebImageURLs_2-model.ttl',
        'input_file': 'NPGWebImageURLs_2.csv',
        'output_file_name': 'NPGWebImageURLs_2'
    },
]


from typing import TextIO

import requests
import time
import json

plot = open('/path/to/file', 'w')
id_list = open('/path/to/file')
api_key = 'the_movie_db_api_key'
headers = {'api_key': api_key}
api_url_base = 'https://api.themoviedb.org/3/movie/'
keywords = '/keywords'
api_url_imdbid = id_list.readline()[:-1]
i = 0
plot.write('[')

while api_url_imdbid:
    i = i + 1
    response = requests.get(api_url_base + api_url_imdbid + keywords, params=headers)
    if response.status_code == 200:
        if "[]" in response.text:
            print('in line ' + str(i) + ' no keywords')
        else:
            plot.write('{"imdb_id":' + api_url_imdbid + '},[' + response.text + "],")
    else:
        print("response status code:" + str(response.status_code))
    time.sleep(0.26)
    print("current line: " + str(i) + "/6250918")
    api_url_imdbid = id_list.readline()[:-1]
plot.write(']')


from django import template
from django.urls import reverse
register = template.Library()

@register.filter
def tag_auth(request,current_path):
    import re
    allowed_url = request.session.get("allowed_url")
    for url in allowed_url:
        if re.match(f'^{url.get("roles__permissions__url")}$', current_path):
            return 'allow'
    else:
        return 'remove'

@register.simple_tag
def url_encode(requset, target_url, cid=''):
    from django.http import QueryDict

    transfer_qdict = QueryDict(mutable=True)
    obj = requset.GET.copy()
    if cid == '':
        target_url = reverse(target_url)
        transfer_qdict['next_url'] = requset.get_full_path()
    else:
        target_url = reverse(target_url, args=(cid,))
        transfer_qdict['next_url'] = requset.path + '?'+obj.urlencode()
    next_url = transfer_qdict.urlencode()
    fullpath = target_url + "?" + next_url
    return fullpath


nintendo_licensee_codes = {
	#Big list! Used in every Nintendo system except: NES (but FDS does use it), N64 (but 64DD does use it), e-Reader, and depending on who you ask Pokemon Mini either has 2P for every game (i.e. if you ask me) or it doesn't use this. Well, I presume Switch does in some way; 3DS seems to be a bit weird, and Wii U uses 4 characters so I guess they ran out of codes with all the indie publishers (most just use this list prepended with 00)
	#Preferred format that I probably don't always use consistently:
	#Company without Co, Inc, Ltd, etc
	#Company (Brand name used by company)
	#Company A / Company B which purchased A or A otherwise got absorbed into
	#Company A / New name of Company A
	#[Company B that merged with company A but kept A's licensee code] Company A e.g. [Bandai] Namco (Namco used the code AF before the merger)

	#00 is invalid and in theory indicates an unlicensed game, but then a lot of official 3DS games (both digital and cart stuff) seem to use it, so... I don't know what's going on there

	#These are definitely what they are, appears in multiple games that are definitely published by that thing
	'01': 'Nintendo',
	'08': 'Capcom',
	'0A': 'Jaleco',
	'0B': 'Coconuts Japan',
	'13': 'Electronic Arts Victor', #EA's Japanese branch (partnered with Victor Interactive), although in 2003 it supposedly went defunct
	'18': 'Hudson Soft',
	'1F': 'Virgin Japan', #Unsure of exact name but it's Virgin but Japanese versions of games. Also found in the Muhammad Ali Heavyweight Boxing prototype (SNES)
	'1M': 'Micro Cabin',
	'1Q': 'TDK Core', #TDK's Japanese game publishing branch
	'29': 'Seta',
	'2L': 'Tamsoft',
	'2N': 'Smilesoft',
	'2P': 'The Pokémon Company',
	'30': 'Viacom',
	'36': 'Codemasters',
	'39': 'Event Evolution Entertainment',
	'41': 'Ubisoft',
	'46': 'System 3', #The old one, used as their licensee code before they started using the other licensee code
	'49': 'Irem',
	'4B': 'Raya Systems',
	'4S': 'Black Pearl Software',
	'4Y': 'Rare',
	'4Z': 'Crave Entertainment', #Resident Evil GBC prototype also has this, but it was developed by HotGen and would have been published by Capcom
	'50': 'Absolute Entertainment',
	'51': 'Acclaim',
	'52': 'Activision',
	'55': 'Hi-Tech Expressions',
	'56': 'LJN',
	'58': 'Mattel',
	'5A': 'Mindscape',
	'5F': 'American Softworks',
	'5G': 'Majesco',
	'5H': 'The 3DO Company',
	'5K': 'Hasbro',
	'5L': 'NewKidCo',
	'5T': 'Cryo Interactive',
	'5X': 'Microids',
	'60': 'Titus',
	'61': 'Virgin',
	'64': 'LucasArts',
	'67': 'Ocean',
	'69': 'Electronic Arts', #nice
	'6B': 'Laser Beam Entertainment', #Publishing arm of Beam Software
	'6F': 'Electro Brain',
	'6H': 'BBC Multimedia',
	'6J': 'Software 2000',
	'6L': 'BAM! Entertainment',
	'6M': 'System 3', #The new one. What the fuck? They didn't even change the spelling or anything, it really is the same company. Gee System 3! How come your mum lets you eat _two_ weiners?
	'6S': 'TDK Mediactive', #TDK's Not-Japan publishing branch
	'6V': 'JoWooD Entertainment',
	'71': 'Interplay',
	'72': 'JVC',
	'75': 'The Sales Curve', #AKA Sales Curve Interactve or SCi (short for Sales Curve with lowercase interactive for some reason)
	'78': 'THQ',
	'79': 'Accolade',
	'7C': 'Microprose', #Seen in Tinhead (Europe) (Proto) (SNES) and F-15 Strike Eagle II (Unknown) (Proto) (Game Boy); which are both prototypes and so therefore unreliable, but they both are indeed by Microprose so this is probably correct
	'7F': 'Kemco',
	'7G': 'Rage Software',
	'7L': 'Simon & Schuster',
	'82': 'Namcot', #Some division of Namco that was apparently necessary to create as a separate thing.... I don't know
	'8B': 'Bullet-Proof Software',
	'8E': 'Character Soft',
	'8F': "I'Max",
	'8P': 'Sega',
	'91': 'Chunsoft',
	'93': 'Bec',
	'95': 'Varie',
	'97': 'Kaneko',
	'9A': 'Nichibutsu',
	'9B': 'Tecmo',
	'9C': 'Imagineer',
	'9H': 'Bottom Up',
	'9M': 'Jaguar', #The sewing machine company
	'9N': 'Marvelous Entertainment', #Before merger in 2003
	'A0': 'Telenet', #Sometimes known as Telenet Japan, or Nippon Telenet, but I think it's fair to just call it "Telenet"
	'A4': 'Konami',
	'A5': 'K Amusement Leasing',
	'A7': 'Takara', #Vast Fame also uses this for most of their GBC bootlegs, interestingly
	'A8': 'Royal Industries', #Also a sewing machine company that is also known as Royal Kyougou, which I guess is just Japanese for Royal Industries
	'AD': 'Toho',
	'AH': 'J-Wing',
	'AL': 'Media Factory',
	'B0': 'Acclaim Japan',
	'B1': 'ASCII',
	'B2': 'Bandai',
	'B3': 'Soft Pro',
	'B4': 'Enix',
	'B6': 'HAL',
	'BA': 'Culture Brain',
	'BB': 'Sunsoft',
	'BC': 'Toshiba EMI',
	'BD': 'Sony Imagesoft',
	'BJ': 'Compile',
	'BL': 'MTO',
	'C0': 'Taito',
	'C1': 'Sunsoft (Chinou Game Series)', #I guess it's an educational branch of Sunsoft? Or it might all be published by Ask Odansha
	'C3': 'Squaresoft',
	'C5': 'Data East',
	'C6': 'Tonkin House',
	'C8': 'Koei',
	'CB': 'Vap',
	'CF': 'Angel',
	'CP': 'Enterbrain',
	'D1': 'Sofel',
	'D9': 'Banpresto',
	'DA': 'Tomy',
	'DD': 'Masaya', #Brand that Nippon Computer Systems uses to distribute games... maybe more adequately called that?
	'DE': 'Human', #Or Human Entertainment
	'DF': 'Altron', #mb2gba uses this too for som reason, as well as a few GBA bootlegs
	'E2': 'Yuutaka', #Or is it spelled Yutaka?
	'E4': 'T&E Soft',
	'E5': 'Epoch',
	'E7': 'Athena',
	'E9': 'Natsume',
	'EB': 'Atlus',
	'FJ': 'Virtual Toys',
	'FQ': 'iQue', #Sort of. Only WarioWare: Touched, Polarium, and Yoshi: Touch & Go use this, the other three iQue games: New Super Mario Bros, Super Mario 64, and Nintendogs (kiosk demo) use Nintendo instead. Those were all 2007 or later, the latter being on the iQue DSi, so it may be something corporate related going on there
	'FR': 'Digital Tainment Pool', #Or DTP Entertainment if you prefer
	'FT': 'Daiwon C&A Holdings',
	'GD': 'Square Enix',
	'GL': 'Gameloft',
	'HF': 'Level5',
	'JS': 'Digital Leisure',
	'KR': 'Krea Medie',
	'KM': 'Deep Silver',
	'RW': 'RealNetworks', #Or GameHouse, or Real Arcade; which are sorta subsidiaries I guess
	'WY': 'WayForward',
	'XN': 'Netflix',

	#Mostly verified, just that companies buy each other out and change their names and things, so these are a bit weird and dependent on what year it is
	'4F': 'U.S. Gold/Eidos', #Eidos purchased U.S. Gold in 1996; also this appears in the Gauntlet DS proto, but that wasn't going to be published by Eidos nor did they develop it
	'5D': 'Midway/Tradewest/Williams', #For all intents and purposes starts off as Bally Midway (it doesn't, but let's not care about pre-1969), purchased by Williams in 1988 (but as "Bally/Midway"), albeit in 1991 started using the name Midway alone, then in 1996 some screwy stuff happened involving WMS and Time Warner, becomes independent in 1998 (but keeps the subsidiary known as "Atari Games" just to confuse me further until 2000) and then now they're sold to Warner Bros and then there's Tradeswest in there somewhere but only up to 1994? Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
	'70': 'Infogrames/Atari, SA', #Infogrames renamed itself to Atari or purchased the brand name or something in 2003 because they could, but are otherwise unrelated to the OG Atari, also purchased and sold off all sorts of companies along the way (e.g. GT Interactive in 1999, Hasbro Interactive in 2001)
	'7D': 'Vivendi/Sierra', #This is weird, because Sierra was merged with Activision in 2010, who at the time were owned by Vivendi, but now they aren't and then they revived the Sierra brand name in 2014. So that's confusing. One of these boxes I have lying around containing a game with this licensee code says Sierra on it, and that was... 2008? So I dunno
	'7J': 'Zoo Digital Publishing', #AKA Zushi Games. See 20 for rambling
	'99': 'Pack-in Video/Victor Interactive/Marvelous Interactive', #Merged with Victor in 1997 and then with Marvelous in 2003, so Victor and Marvelous probably used different codes before then (or weren't involved with Nintendo)
	'AF': '[Bandai] Namco', #Namco games have always used this, but when they merged with Bandai (Sep 2005), they kept this code (so newer games like that Code Geass DS one use this code as well), which is interesting because Square Enix doesn't reuse Squaresoft or Enix's licensee codes and gets a new one

	#Welcome to confusing town, population these
	'20': 'Destination Software', #This is a confusing load of fuck, and I hate this entire company. So basically you also have 7J, and at first you'd think 20 is the UK/Europe Zoo Digital Publishing (aka Zoo Games) (aka Zushi Games), and 7J is the USA Destination Software (aka DSI Games). But then that turns out to not be the case. They're sort of the same company and have been traded around and bought out and sold multiple times. Thanks I hate it.
	'4Q': 'Disney/Buena Vista Interactive', #What's maximum spooky is that this also shows up in the Spiderman: Friend or Foe trailer, but that was 2007 and Disney hadn't purchased Marvel yet. Disney Interactive Studios spun off Buena Vista Interactive from 2003 to 2007 but that still doesn't explain Spiderman. It's not in any of the other DS downloadable videos. What the crap?
	'54': 'Take-Two Interactive', #Oof this one's a really fun one. Some documentation says "Konami/GameTek" which is just garbage because Konami has nothing to do with this, but some older games using this code like Wheel of Fortune on SNES and the InfoGenius Productivity Pak on GB are indeed published under GameTek. Where it gets fun is that GameTek became Take-Two Interactive at some point, and Duke Nukem Advance uses this code too and was published under the Take-Two Interactive name.. but then Dora the Explorer: Dora's World Adventures also uses this code but was published under Global Star Software, which was a company that became 2K Play which is then a branch of 2K Games and if you've lost track of everything I don't blame you and I just spent several tens of minutes googling around for info on Dora the Explorer why do I do this to myself
	#Anyway this one just seems to cover everything that's owned by Take-Two Interactive
	'5Z': 'Classified Games', #Also appears in the Card Shark (SNES) proto, which as far as I can tell was developed by someone named Bonsai and involved someone named Bicycle as well. Seems like there's a mixup here with Conspiracy Entertainment? Are they the same company?
	'AC': 'Toei Animation', #Used in some EXTREMELY obscure "Waiwai Check" games for Satellaview, which according to the title screen of one of them, are made by Hori Electric? Well I'm confused; Tooyama no Kinsan Space Chou - Mr. Gold is actually published by Toei though (apparently)
	'CA': 'Konami (Ultra Games)', #Definitely Konami but might not be Ultra Games? Published Batman Returns SNES in Japan (but not elsewhere), and Parodius (Europe) for Game Boy
	'CE': 'Pony Canyon/FCI', #Seems to be Fujisankei Communications International (as FCI Inc) for USA releases, Pony Canyon for Japanese releases, both are owned by Fujisankei Communications Group

	#So far, only seen once (in a reliable source, i.e. not protos or anything like that)
	'09': 'Hot-B', #Ginga - Card & Puzzle Collection (Japan) (En,Ja)
	'19': 'Bandai (B-AI)', #Pingu - Sekai de 1ban Genki na Penguin (Japan)
	'1P': 'Creatures', #Chee-Chai Alien (Japan) (GBC)
	'35': 'Hector', #California Games II (USA) (might be just Hect?) (game was published by DTMC in USA and Hect in Japan, but DTMC seems to be related to Hect and may be a subdivision/branch?)
	'47': 'Spectrum Holobyte', #Star Trek: The Next Generation: Future's Past (USA) (SNES)
	'4A': 'Gakken', #Titanic Mystery - Ao no Senritsu
	'53': 'American Sammy', #Ys III (SNES); also Ninja Taro (USA) (Beta) (GB); dunno what the difference is between this and non-American Sammy
	'5Q': 'Lego', #Lego Island 2: The Brickster's Revenge
	'6E': 'Elite Systems', #Might and Magic II (Europe) (SNES)
	'6P': 'Ravensburger Interactive', #Fix & Foxi - Episode 1 - Lupo (Europe) (En,Fr,De) (GBC)
	'6R': 'Sound Source', #Extreme Sports with the Berenstain Bears (USA, Europe) (En,Fr,De,Es,It) (GBC)
	'8C': 'Vic Tokai', #Zerd no Densetsu (Japan)
	'8N': 'Success', #Minna no Soft Series - Tetris Advance (Japan)
	'A6': 'Kawada', #Othello (FDS)
	'A9': 'Technos Japan', #Shin Nekketsu Kouha - Kunio-tachi no Banka (Japan) [T-En by Aeon Genesis v1.00] (SNES)
	'BF': 'Sammy', #Gindama Oyakata no Pachinko Hisshouhou (Japan)
	'C7': 'East Cube', #Reflect World (FDS) (did they even make any other games?)
	'CU': 'Marvelous AQL', #Senran Kagura Burst (Europe) (3DS)
	'D4': 'Ask Kodansha', #Magna Braban - Henreki no Yuusha
	'D6': 'Naxat Soft', #Spriggan Powered BS - Prelude
	'DB': 'Hiro', #Daisenryaku (Japan) (Gameboy)
	'E1': 'Towa Chiki', #Taikyoku Renju
	'E8': 'Asmik', #Dokapon Gaiden - Honoo no Audition (Japan) (merged with Ace to become Asmik Ace in 1998)
	'EA': 'King Records', #Azumanga Daioh Advance
	'EC': 'Epic / Sony Records', #Jerry Boy (SNES)
	'EE': 'Information Global Services', #Super Shadow of the Beast (USA) (Proto) (SNES)
	'F3': 'Extreme Entertainment', #Super Solitaire (USA) (En,Fr,De,Es,It) (SNES)
	'FE': 'Davidson & Associates', #Math Blaster - Episode 1 (USA) (SNES)
	'GN': 'OG International', #Cartoon Network Punch Time Explosion XL (Europe) (Wii)
	'GY': 'The Game Factory', #Code Lyoko (USA) (DS) (might be "The American Game Factory", both names are used on the box so I dunno)
	'H4': 'SNK Playmore', #Doki Doki Majo Shinpan (DS)
	'HL': 'Frontier Developments', #LostWinds (USA) (WiiWare)
	'HN': 'High Voltage', #VIP Casino Blackjack (USA) (WiiWare)
	'JT': 'Empty Clip Studios', #Groovin' Blocks (USA) (WiiWare) (retail release is by Zoo)
	'JY': 'Steel Penny', #Bruiser and Scratch (USA) (WiiWare)
	'NR': 'Destineer', #Giana Sisters DS (USA)
	'PV': 'Pan Vision Games', #Miffy's World (Europe) (DS)
	'RG': 'Romino Games', #Swords & Soldiers II (Wii U)
	'S3': 'Genius Sonority', #The Denpa Men demo (3DS)
	'TL': 'Telltale Games', #Strong Badia the Free series (WiiWare)
	'QU': 'QubicGames', #AiRace Speed demo (3DS)
	'UZ': 'Circle Entertainment', #Sweet Memories: Blackjack (3DS)
	'WR': 'Warner Bros', #Scribblenauts and Super Scribblenauts (Europe) (DS) (or Warner Bros Interactive Entertainment if you prefer to be overly specific)
	'YV': 'KnapNok Games', #Affordable Space Adventures (Wii U)

	#So far only seen once, but probably only would ever be seen once
	'62': 'Maxis', #SimAnt - The Electronic Ant Colony (USA) (SNES) (only release they ever published themselves as far as Nintendo platforms are concerned)
	'9Z': 'Crunchyroll', #Crunchyroll app on both Wii and Wii U, so I guess that's published by Crunchyroll, well the Wii U version's metadata says so anyway
	'WB': 'Google', #YouTube app on Wii and Wii U
	'WD': 'Amazon', #Amazon Instant Video (WiiWare)
	'XB': 'Hulu', #Hulu Plus (WiiWare)
}


#! /usr/bin/python2.7
# -*- coding: utf8 -*-

import sys
import os, subprocess, shlex, re, gzip
# Just for DEBUG
#os.chdir('/Users/sofiasilva/GitHub/cool_bgp_stats')
from get_file import get_file
import bgp_rib
import pickle
import radix
import datetime, calendar
import pandas as pd
import hashlib
import ipaddress

# For some reason in my computer os.getenv('PATH') differs from echo $PATH
# /usr/local/bin is not in os.getenv('PATH')
# it also works in matong
bgpdump = '/usr/local/bin/bgpdump'

class BGPDataHandler:
    DEBUG = False
    files_path = ''
    KEEP = False
    RIBfiles = False
    COMPRESSED = False

    # STRUCTURES WITH CURRENT ROUTING DATA
    # Data Frame containing routing info from RIB file or file with 'show ip bgp' output
    bgp_data = pd.DataFrame()

   # Radix indexed by routed IPv4 prefix containing the indexes of the rows in
    # the bgp_data Data Frame that contain info of BGP announcements of the IPv4 prefix
    ipv4_prefixes_indexes_radix = radix.Radix()

    # Radix indexed by routed IPv6 prefix containing the indexes of the rows in
    # the bgp_data Data Frame that contain info of BGP announcements of the IPv6 prefix
    ipv6_prefixes_indexes_radix = radix.Radix()

    # Dictionary indexed by AS containing all the prefixes originated by each AS
    ASes_originated_prefixes_dic = dict()

    # Dictionary indexed by AS containing all the prefixes propagated by each AS
    ASes_propagated_prefixes_dic = dict()

    # Numeric variable with the longest IPv4 prefix length
    ipv4_longest_pref = -1

    # Numeric variable with the longest IPv6 prefix length
    ipv6_longest_pref = -1

    # STRUCTURES WITH TEMPORAL DATA
    # Radix indexed by routed IPv4 prefix containing as values dictionaries
    # with the following keys:
    # * periodsSeen - the value for this key is a list of tuples representing
    # periods of time during which the corresponding IPv4 prefix was seen
    # Each tuple has the format (startDate, endDate)
    # * firstSeen - the value for this key is the first date in which the
    # IPv4 prefix was seen
    # * lastSeen - the value for this key is the last date in which the
    # IPv4 prefix was seen
    # * totalDays - the value for this key is the number of days during
    # which the IPv4 prefix was seen
    ipv4_prefixesDates_radix = radix.Radix()

    # Radix indexed by routed IPv6 prefix containing as values dictionaries
    # with the following keys:
    # * periodsSeen - the value for this key is a list of tuples representing
    # periods of time during which the corresponding IPv6 prefix was seen
    # Each tuple has the format (startDate, endDate)
    # * firstSeen - the value for this key is the first date in which the
    # IPv6 prefix was seen
    # * lastSeen - the value for this key is the last date in which the
    # IPv6 prefix was seen
    # * totalDays - the value for this key is the number of days during
    # which the IPv6 prefix was seen
    ipv6_prefixesDates_radix = radix.Radix()

    # Dictionary indexed by origin ASN (ASN that originates at least one prefix)
    # It contains as values dictionaries with the following keys:
    # * periodsSeen - the value for this key is a list of tuples representing
    # periods of time during which the corresponding ASN originated prefixes
    # Each tuple has the format (startDate, endDate)
    # * firstSeen - the value for this key is the first date in which the
    # ASN originated prefixes
    # * lastSeen - the value for this key is the last date in which the
    # ASN originated prefixes
    # * totalDays - the value for this key is the number of days during
    # which the ASN originated prefixes
    originASesDates_dict = dict()

    # Dictionary indexed by middle ASN (ASN that propagates at least one prefix)
    # It contains as values dictionaries with the following keys:
    # * periodsSeen - the value for this key is a list of tuples representing
    # periods of time during which the corresponding ASN prpagated prefixes
    # Each tuple has the format (startDate, endDate)
    # * firstSeen - the value for this key is the first date in which the
    # ASN propagated prefixes
    # * lastSeen - the value for this key is the last date in which the
    # ASN propagated prefixes
    # * totalDays - the value for this key is the number of days during
    # which the ASN propagated prefixes
    middleASesDates_dict = dict()

    # When we instantiate this class we set the variable with the path to the
    # folder we will use to store files (files_path), we set a boolean variable
    # specifying whether we want to KEEP the intermediate files generated by
    # different functions, we set another boolean variable specifying whether
    # the routing files we will be working with are RIB files (if False we assume
    # they are outputs of the 'show ip bgp' command) and we set another boolean
    # variable specifying whether the routing files we will be working with are
    # COMPRESSED
    def __init__(self, DEBUG, files_path, KEEP, RIBfiles, COMPRESSED):
        self.DEBUG = DEBUG
        self.files_path = files_path
        self.KEEP = KEEP
        self.RIBfiles = RIBfiles
        self.COMPRESSED = COMPRESSED

        sys.stderr.write("BGPDataHandler instantiated successfully! Remember to load the data structures.\n")

    # This function loads the class variables ipv4_prefixesDates and ipv6_prefixesDates
    # from a previously generated pickle file containing Radixes indexed by
    # routed prefix containing as values dictionaries with the following keys:
    # * periodsSeen - the value for this key is a list of tuples representing
    # periods of time during which the corresponding prefix was seen
    # Each tuple has the format (startDate, endDate)
    # * firstSeen - the value for this key is the first date in which the
    # prefix was seen
    # * lastSeen - the value for this key is the last date in which the
    # prefix was seen
    # * totalDays - the value for this key is the number of days during
    # which the prefix was seen
    def loadPrefixDatesFromFiles(self, ipv4_prefixesDates_file, ipv6_prefixesDates_file):
        if ipv4_prefixesDates_file != '':
            self.ipv4_prefixesDates_radix = pickle.load(open(ipv4_prefixesDates_file, 'rb'))
            sys.stderr.write("Radix with dates in which IPv4 prefixes were seen was loaded successfully!\n")

        if ipv6_prefixesDates_file != '':
            self.ipv6_prefixesDates_radix = pickle.load(open(ipv6_prefixesDates_file, 'rb'))
            sys.stderr.write("Radix with dates in which IPv6 prefixes were seen was loaded successfully!\n")

    # This function loads the class variable originASesDates from a previously
    # generated pickle file containing a originASesDates dictionary
    def loadOriginASesDatesFromFile(self, originASesDates_file):
        if originASesDates_file != '':
            self.originASesDates_dict = pickle.load(open(originASesDates_file, 'rb'))
            sys.stderr.write("Dictionary with dates in which ASNs originated prefixes was loaded successfully!\n")

    # This function loads the class variable middleASesDates from a previously
    # generated pickle file containing a middleASesDates dictionary
    def loadMiddleASesDatesFromFile(self, middleASesDates_file):
        if middleASesDates_file != '':
            self.middleASesDates_dict = pickle.load(open(middleASesDates_file, 'rb'))
            sys.stderr.write("Dictionary with dates in which ASNs propagated prefixes was loaded successfully!\n")

    # This function loads the data structures of the class from previously
    # generated pickle files containing the result of already processed routing data
    def loadStructuresFromFiles(self, bgp_data_file, ipv4_prefixes_indexes_file,\
                                ipv6_prefixes_indexes_file, ASes_originated_prefixes_file,\
                                ASes_propagated_prefixes_file):

        self.bgp_data = pickle.load(open(bgp_data_file, "rb"))
        self.ipv4_prefixes_indexes_radix = pickle.load(open(ipv4_prefixes_indexes_file, "rb"))
        self.ipv6_prefixes_indexes_radix = pickle.load(open(ipv6_prefixes_indexes_file, "rb"))
        self.ASes_originated_prefixes_dic = pickle.load(open(ASes_originated_prefixes_file, "rb"))
        self.ASes_propagated_prefixes_dic = pickle.load(open(ASes_propagated_prefixes_file, "rb"))
        self.setLongestPrefixLengths()
        sys.stderr.write("Class data structures were loaded successfully!\n")
        return True

    # This function processes the routing data contained in the files to which
    # the URLs in the urls_file point, and loads the data structures of the class
    # with the results from this processing
    def loadStructuresFromURLSfile(self, urls_file):
        bgp_data, ipv4_prefixes_indexes_radix, ipv6_prefixes_indexes_radix,\
            ASes_originated_prefixes_dic, ASes_propagated_prefixes_dic,\
            ipv4_longest_pref, ipv6_longest_pref  =\
                        self.processMultipleFiles(files_list=urls_file,\
                                                isList=False, containsURLs=True)

        self.bgp_data = bgp_data
        self.ipv4_prefixes_indexes_radix = ipv4_prefixes_indexes_radix
        self.ipv6_prefixes_indexes_radix = ipv6_prefixes_indexes_radix
        self.ASes_originated_prefixes_dic = ASes_originated_prefixes_dic
        self.ASes_propagated_prefixes_dic = ASes_propagated_prefixes_dic

        if ipv4_longest_pref != -1:
            self.ipv4_longest_pref = ipv4_longest_pref
        else:
            self.ipv4_longest_pref = 32
        if ipv6_longest_pref != -1:
            self.ipv6_longest_pref = ipv6_longest_pref
        else:
            self.ipv6_longest_pref = 64

        sys.stderr.write("Class data structures were loaded successfully!\n")
        return True

    # This function processes the routing data contained in the routing_file
    # and loads the data structures of the class with the results from this processing
    def loadStructuresFromRoutingFile(self, routing_file):
        readable_file_name =  self.getReadableFile(routing_file, False)

        if readable_file_name != '':
            bgp_data, ipv4_prefixes_indexes_radix, ipv6_prefixes_indexes_radix,\
                ASes_originated_prefixes_dic, ASes_propagated_prefixes_dic,\
                ipv4_longest_pref, ipv6_longest_pref =\
                                    self.processReadableDF(readable_file_name)

            self.bgp_data = bgp_data
            self.ipv4_prefixes_indexes_radix = ipv4_prefixes_indexes_radix
            self.ipv6_prefixes_indexes_radix = ipv6_prefixes_indexes_radix
            self.ASes_originated_prefixes_dic = ASes_originated_prefixes_dic
            self.ASes_propagated_prefixes_dic = ASes_propagated_prefixes_dic

            if ipv4_longest_pref != -1:
                self.ipv4_longest_pref = ipv4_longest_pref
            else:
                self.ipv4_longest_pref = 32
            if ipv6_longest_pref != -1:
                self.ipv6_longest_pref = ipv6_longest_pref
            else:
                self.ipv6_longest_pref = 64

            sys.stderr.write("Class data structures were loaded successfully!\n")
            return True
        else:
            sys.stderr.write("Could not process routing file.\n")
            return False

    # This function processes the routing data contained in the archive folder
    # provided, and loads the data structures of the class with the results
    # from this processing
    def loadStructuresFromArchive(self, archive_folder, extension, startDate, endDate):
        historical_files = self.getPathsToHistoricalData(archive_folder, extension)

        if historical_files == '':
            sys.stderr.write("Archive is empty!\n")
            return False

        mostRecent_routing_file  =\
                        self.getMostRecentFromHistoricalList(historical_files, endDate)

        mostRecent_readable = self.getReadableFile(mostRecent_routing_file,\
                                False)

        # In order for the most recent file not to be processed twice,
        # we load data from this file into the prefixesDates Radixes
        # now that we have the readable file available
        self.loadHistoricalDataFromFile(mostRecent_readable, True)

        # We then load the rest of the data structures
        bgp_data, ipv4_prefixes_indexes_radix, ipv6_prefixes_indexes_radix,\
            ASes_originated_prefixes_dic, ASes_propagated_prefixes_dic,\
            ipv4_longest_pref, ipv6_longest_pref =\
                                self.processReadableDF(mostRecent_readable)

        self.bgp_data = bgp_data
        self.ipv4_prefixes_indexes_radix = ipv4_prefixes_indexes_radix
        self.ipv6_prefixes_indexes_radix = ipv6_prefixes_indexes_radix
        self.ASes_originated_prefixes_dic = ASes_originated_prefixes_dic
        self.ASes_propagated_prefixes_dic = ASes_propagated_prefixes_dic

        if ipv4_longest_pref != -1:
            self.ipv4_longest_pref = ipv4_longest_pref
        else:
            self.ipv4_longest_pref = 32
        if ipv6_longest_pref != -1:
            self.ipv6_longest_pref = ipv6_longest_pref
        else:
            self.ipv6_longest_pref = 64

        sys.stderr.write("Class data structures were loaded successfully!\n")

        # Finally, we load the prefixesDates Radixes and the originASesDates
        # dictionary with the rest of the routing files from the archive,
        # providing the name of the most recent file in order for it to be skipped.
        self.loadDatesStructures(historical_files, startDate, endDate, mostRecent_routing_file)
        sys.stderr.write("Radixes with dates in which prefixes were seen were loaded successfully!\n")

        return True

    # This function returns a path to the most recent file in the provided list
    # of historical files
    def getMostRecentFromHistoricalList(self, historical_files, endDate):
        files_list_obj = open(historical_files, 'r')

        mostRecentDate = 0
        mostRecentFile = ''

        for line in files_list_obj:
            if not line.startswith('#') and line.strip() != '':
                date = self.getDateFromFileName(line.strip())

                if date > mostRecentDate:
                    # We add 1 to the endDate because the files in the archive
                    # have routing data for the day before of the date in the
                    # name of the file
                    if endDate == '' or (endDate != '' and date <= int(endDate)+1):
                        mostRecentDate = date
                        mostRecentFile = line.strip()

        return mostRecentFile

    def getDateFromFileName(self, filename):
        date = ''

        dates = re.findall('[1-2][9,0][0,1,8,9][0-9]-[0-1][0-9]-[0-3][0-9]',\
                    filename)

        if len(dates) > 0:
            date = int(dates[0][0:4]+dates[0][5:7]+dates[0][8:10])
        else:
            dates = re.findall('[1-2][9,0][0,1,8,9][0-9][0-1][0-9][0-3][0-9]',\
                        filename)
            if len(dates) > 0:
                date = int(dates[0])
        return date

    # This function loads the prefixesDates Radixes (class variables) and
    # the originASesDates dictionary with the routing data from the files
    # listed in the historical_files file and which have a date more recent
    # than startDate in case startDate is provided. If startDate is not
    # provided, all the files listed in the historical_files file will be processed.
    def loadDatesStructures(self, historical_files, startDate, endDate, mostRecent):

        files_list_obj = open(historical_files, 'r')

        i = 0
        for line in files_list_obj:
            line = line.strip()
            if line == mostRecent:
                continue

            if startDate != '' or endDate != '':
                file_date = self.getDateFromFileName(line)

                if file_date == '' or\
                    (startDate != '' and int(file_date) < int(startDate)) or\
                    (endDate != '' and int(file_date) > int(endDate)+1):
                    # We add 1 to the endDate because the files in the archive
                    # have routing data for the day before of the date in the
                    # name of the file
                    continue

            if not line.startswith('#') and line != '':
                 # If we work with several routing files
                sys.stderr.write("Starting to work with %s\n" % line)

                self.loadHistoricalDataFromFile(line, False)

            i += 1
            if self.DEBUG and i > 1:
                break

    def updateDatesDict(self, dictionary, date):
        if date < dictionary['firstSeen']:
            dictionary['firstSeen'] = date

        if date > dictionary['lastSeen']:
            dictionary['lastSeen'] = date

        dateReady = False
        for period in dictionary['periodsSeen']:
            if date >= period[0]:
                if date <= period[1]:
                    dateReady = True
                    continue
                elif date == period[1]+1:
                    dictionary['periodsSeen'].remove(period)
                    dictionary['periodsSeen'].append((period[0], date))
                    dictionary['totalDays'] += 1
                    dateReady = True
        if not dateReady:
            dictionary['periodsSeen'].append((date, date))
            dictionary['totalDays'] = 1

    # This function loads the prefixesDates Radixes and the originASesDates
    # dictionary with the routing data from the routing_file provided
    def loadHistoricalDataFromFile(self, routing_file, isReadable):
        prefixes, originASes, middleASes, date =\
                        self.getPrefixesASesAndDate(routing_file, isReadable)

        for pref in prefixes:
            network = ipaddress.ip_network(unicode(pref, 'utf-8'))

            if network.version == 4:
                prefixesDates = self.ipv4_prefixesDates_radix
            else:
                prefixesDates = self.ipv6_prefixesDates_radix

            pref_node = prefixesDates.search_exact(pref)
            if pref_node is not None:
                self.updateDatesDict(pref_node.data, date)
            else:
                pref_node = prefixesDates.add(pref)
                pref_node.data['periodsSeen'] = [(date, date)]
                pref_node.data['firstSeen'] = date
                pref_node.data['lastSeen'] = date
                pref_node.data['totalDays'] = 1

        for asn in originASes:
            if asn is None or asn == 'nan':
                continue
            elif '{' in asn:
                # If the asn field contains a bracket ({}), there is an as-set
                # in first place in the AS path, therefore, we split it
                # (leaving the brackets out) and consider each AS separately.
                asnList = asn[1:-1] .split(',')
                for asn in asnList:
                    if asn in self.originASesDates_dict:
                        self.updateDatesDict(self.originASesDates_dict[asn], date)
                    else:
                        self.originASesDates_dict[asn] = dict()
                        self.originASesDates_dict[asn]['periodsSeen'] = [(date, date)]
                        self.originASesDates_dict[asn]['firstSeen'] = date
                        self.originASesDates_dict[asn]['lastSeen'] = date
                        self.originASesDates_dict[asn]['totalDays'] = 1
            else:
                asn = int(asn)
                if asn in self.originASesDates_dict:
                    self.updateDatesDict(self.originASesDates_dict[asn], date)
                else:
                    self.originASesDates_dict[asn] = dict()
                    self.originASesDates_dict[asn]['periodsSeen'] = [(date, date)]
                    self.originASesDates_dict[asn]['firstSeen'] = date
                    self.originASesDates_dict[asn]['lastSeen'] = date
                    self.originASesDates_dict[asn]['totalDays'] = 1

        for asn in middleASes:
            if asn is None or asn == 'nan':
                continue
            elif '{' in asn:
                asnList = asn[1:-1] .split(',')
                for asn in asnList:
                    if asn in self.middleASesDates_dict:
                        self.updateDatesDict(self.middleASesDates_dict[asn], date)
                    else:
                        self.middleASesDates_dict[asn] = dict()
                        self.middleASesDates_dict[asn]['periodsSeen'] = [(date, date)]
                        self.middleASesDates_dict[asn]['firstSeen'] = date
                        self.middleASesDates_dict[asn]['lastSeen'] = date
                        self.middleASesDates_dict[asn]['totalDays'] = 1
            else:
                asn = int(asn)
                if asn in self.middleASesDates_dict:
                    self.updateDatesDict(self.middleASesDates_dict[asn], date)
                else:
                    self.middleASesDates_dict[asn] = dict()
                    self.middleASesDates_dict[asn]['periodsSeen'] = [(date, date)]
                    self.middleASesDates_dict[asn]['firstSeen'] = date
                    self.middleASesDates_dict[asn]['lastSeen'] = date
                    self.middleASesDates_dict[asn]['totalDays'] = 1

    # This function returns a list of prefixes for which the routing_file has
    # announcements, a list of the origin ASes included in the routing_file,
    # a list of the middle ASes included in the routing file
    # and the date of the routing file.
    # The routing file is assumed to include routing data for a single day,
    # therefore the date is taken from the timestamp of the first row in the
    # bgp_df DataFrame.
    def getPrefixesASesAndDate(self, routing_file, isReadable):
        if not isReadable:
            readable_file_name = self.getReadableFile(routing_file, False)
        else:
            readable_file_name = routing_file

        if readable_file_name == '':
            return [], [], ''

        bgp_df = pd.read_table(readable_file_name, header=None, sep='|',\
                                index_col=False, usecols=[1,3,5,6,7],\
                                names=['timestamp',\
                                        'peer',\
                                        'prefix',\
                                        'ASpath',\
                                        'origin'])

        if self.DEBUG:
            bgp_df = bgp_df[0:10]

        date = datetime.datetime.utcfromtimestamp(bgp_df['timestamp'].tolist()[0]).strftime('%Y%m%d')

        # To get the origin ASes and middle ASes we split the ASpath column
        paths_parts = bgp_df.ASpath.str.rsplit(' ', n=1, expand=True)

        return set(bgp_df['prefix'].tolist()),\
                set(paths_parts[1].tolist()),\
                set([item for sublist in paths_parts[0].tolist() for item in\
                        str(sublist).split()]), date

    # This function downloads and processes all the files in the provided list.
    # The boolean variable containsURLs must be True if the files_list is a list
    # of URLs or False if it is a list of paths
    def processMultipleFiles(self, files_list, isList, containsURLs):
        if not isList:
            files_list = open(files_list, 'r')

        bgp_data = pd.DataFrame()
        ipv4_prefixes_indexes_radix = radix.Radix()
        ipv6_prefixes_indexes_radix = radix.Radix()
        ASes_originated_prefixes_dic = dict()
        ASes_propagated_prefixes_dic = dict()
        ipv4_longest_pref = -1
        ipv6_longest_pref = -1

        i = 0
        for line in files_list:
            if not line.startswith('#') and line.strip() != '':
                # If we work with several routing files
                sys.stderr.write("Starting to work with %s\n" % line)

                # We obtain partial data structures
                if containsURLs:
                    readable_file_name =  self.getReadableFile(line.strip(), True)

                    if readable_file_name == '':
                        continue

                    bgp_data_partial, ipv4_prefixes_indexes_radix_partial,\
                        ipv6_prefixes_indexes_radix_partial,\
                        ASes_originated_prefixes_dic_partial,\
                        ASes_propagated_prefixes_dic_partial,\
                        ipv4_longest_pref_partial, ipv6_longest_pref_partial =\
                                self.processReadableDF(readable_file_name)
                else:
                    readable_file_name =  self.getReadableFile(line.strip(), False)

                    if readable_file_name == '':
                        continue

                    bgp_data_partial, ipv4_prefixes_indexes_radix_partial,\
                        ipv6_prefixes_indexes_radix_partial,\
                        ASes_originated_prefixes_dic_partial,\
                        ASes_propagated_prefixes_dic_partial,\
                        ipv4_longest_pref_partial, ipv6_longest_pref_partial =\
                                self.processReadableDF(readable_file_name)

                # and then we merge them into the general data structures
                bgp_data = pd.concat([bgp_data, bgp_data_partial])

                for prefix in ipv4_prefixes_indexes_radix_partial.prefixes():
                    node_partial = ipv4_prefixes_indexes_radix_partial.search_exact(prefix)
                    node_gral= ipv4_prefixes_indexes_radix.search_exact(prefix)
                    if node_gral is not None:
                        node_gral.data['indexes'].update(list(node_partial.data['indexes']))
                    else:
                        node_gral = ipv4_prefixes_indexes_radix.add(prefix)
                        node_gral.data['indexes'] = node_partial.data['indexes']

                for prefix in ipv6_prefixes_indexes_radix_partial.prefixes():
                    node_partial = ipv6_prefixes_indexes_radix_partial.search_exact(prefix)
                    node_gral= ipv6_prefixes_indexes_radix.search_exact(prefix)
                    if node_gral is not None:
                        node_gral.data['indexes'].update(list(node_partial.data['indexes']))
                    else:
                        node_gral = ipv6_prefixes_indexes_radix.add(prefix)
                        node_gral.data['indexes'] = node_partial.data['indexes']

                for aut_sys, prefixes in ASes_originated_prefixes_dic_partial.iteritems():
                    if aut_sys in ASes_originated_prefixes_dic.keys():
                        ASes_originated_prefixes_dic[aut_sys].update(list(prefixes))
                    else:
                        ASes_originated_prefixes_dic[aut_sys] = prefixes

                for aut_sys, prefixes in ASes_propagated_prefixes_dic_partial.iteritems():
                    if aut_sys in ASes_propagated_prefixes_dic.keys():
                        ASes_propagated_prefixes_dic[aut_sys].update(list(prefixes))
                    else:
                        ASes_propagated_prefixes_dic[aut_sys] = prefixes

                if ipv4_longest_pref_partial > ipv4_longest_pref:
                    ipv4_longest_pref = ipv4_longest_pref_partial

                if ipv6_longest_pref_partial > ipv6_longest_pref:
                    ipv6_longest_pref = ipv6_longest_pref_partial

            i += 1
            if self.DEBUG and i > 1:
                break

        if not isList:
            files_list.close()

        return bgp_data, ipv4_prefixes_indexes_radix, ipv6_prefixes_indexes_radix,\
            ASes_originated_prefixes_dic, ASes_propagated_prefixes_dic,\
            ipv4_longest_pref, ipv6_longest_pref

    # This function converts a file containing the output of the 'show ip bgp' command
    # to a file in the same format used for BGPDump outputs
    def convertBGPoutput(self, routing_file):
        output_file_name = '%s/%s.readable' % (self.files_path, '.'.join(routing_file.split('/')[-1].split('.')[:-1]))
        output_file = open(output_file_name, 'w')

        i = 0
        # load routing table info  (the next loop does it automatically)
        for entry_n, bgp_entry in enumerate(bgp_rib.BGPRIB.parse_cisco_show_ip_bgp_generator(routing_file)):
            date = bgp_entry[8]
#           date_part = str(date)[0:8]
#           time_part = str(date)[8:12]
            timestamp = calendar.timegm(datetime.datetime.strptime(date, "%Y%m%d%H%M").timetuple())
            next_hop = bgp_entry[2]
            prefix = bgp_entry[0]
            as_path = bgp_entry[6]

            if as_path:
                nextas = as_path[0]
            else:
            	nextas = ''

            if bgp_entry[7] == 'i':
                origin = "IGP"
            elif bgp_entry[7] == 'e':
                origin = "EGP"
            elif bgp_entry[7] == "?":
                origin = "INCOMPLETE"
            else:
                sys.stderr.write("Found invalid prefix at bgp entry %s, with content %s, on file %s\n" %(entry_n, bgp_entry, routing_file))
            	# ignore this line and continue
                continue

            # save information

            #the order for each line is
            #TABLE_DUMP2|date|B|nexthop|NextAS|prefix|AS_PATH|Origin
            output_file.write('TABLE_DUMP|'+str(timestamp)[:-2]+'|B|'+next_hop+'|'+nextas+'|'+prefix+'|'+" ".join(as_path)+'|'+origin+'\n')

            i += 1
            if self.DEBUG and i > 10:
                break

        output_file.close()

        return output_file_name

    # This function processes a readable file with routing info
    # putting all the info into a Data Frame
    def processReadableDF(self, readable_file_name):

        ipv4_prefixes_indexes_radix = radix.Radix()
        ipv6_prefixes_indexes_radix = radix.Radix()
        ASes_originated_prefixes_dic = dict()
        ASes_propagated_prefixes_dic = dict()

        ipv4_longest_prefix = -1
        ipv6_longest_prefix = -1

        bgp_df = pd.read_table(readable_file_name, header=None, sep='|',\
                                index_col=False, usecols=[1, 3,5,6,7],\
                                names=['timestamp',\
                                        'peer',\
                                        'prefix',\
                                        'ASpath',\
                                        'origin'])

        if bgp_df.shape[0] > 0:

            if self.DEBUG:
                bgp_df = bgp_df[0:10]

            # We create an index that is unique even amongst different routing files
            # so that we can merge partial data structures into a single structure
            file_id = hashlib.md5(readable_file_name).hexdigest()
            bgp_df['source_file'] = '%s_' % file_id
            bgp_df['index'] = bgp_df.index.astype(str)
            bgp_df['index'] = bgp_df['source_file'] + bgp_df['index']
            bgp_df.index = bgp_df['index']

             # We add a column to the Data Frame with the corresponding date
            bgp_df['date'] = bgp_df.apply(lambda row: datetime.datetime.utcfromtimestamp(row['timestamp']).strftime('%Y%m%d'), axis=1)

            ASpath_parts = bgp_df.ASpath.str.rsplit(' ', n=1, expand=True)
            bgp_df['middleASes'] = ASpath_parts[0]
            bgp_df['originAS'] = ASpath_parts[1]

            for prefix, prefix_subset in bgp_df.groupby('prefix'):
                network = ipaddress.ip_network(unicode(prefix, 'utf-8'))
                if network.version == 4:
                    if network.prefixlen > ipv4_longest_prefix:
                        ipv4_longest_prefix = network.prefixlen
                    prefixes_indexes_radix = ipv4_prefixes_indexes_radix

                else:
                    if network.prefixlen > ipv6_longest_prefix:
                        ipv6_longest_prefix = network.prefixlen
                    prefixes_indexes_radix = ipv6_prefixes_indexes_radix

                node = prefixes_indexes_radix.add(prefix)
                node.data['indexes'] = set(prefix_subset.index)

                for middleASes in prefix_subset['middleASes']:
                    for asn in middleASes.split():
                        asn = int(asn)
                        if asn in ASes_propagated_prefixes_dic.keys():
                            if prefix not in ASes_propagated_prefixes_dic[asn]:
                                ASes_propagated_prefixes_dic[asn].add(prefix)
                        else:
                            ASes_propagated_prefixes_dic[asn] = set([prefix])

            for originAS, originAS_subset in bgp_df.groupby('originAS'):
                originAS = int(originAS)
                ASes_originated_prefixes_dic[originAS] = set(originAS_subset['prefix'])

            if not self.KEEP:
                try:
                    os.remove(readable_file_name)
                except OSError:
                    pass

        return bgp_df, ipv4_prefixes_indexes_radix, ipv6_prefixes_indexes_radix,\
                ASes_originated_prefixes_dic, ASes_propagated_prefixes_dic,\
                ipv4_longest_prefix, ipv6_longest_prefix

    # This function downloads a routing file if the source provided is a URL
    # If the file is COMPRESSED, it is unzipped
    # and finally it is processed using BGPdump if the file is a RIBfile
    # or using the functions provided by get_rib.py is the file contains the
    # output of the 'show ip bgp' command
    # The path to the resulting readable file is returned
    def getReadableFile(self, source, isURL):

        source_filename = source.split('/')[-1]

        # If a routing file is not provided, download it from the provided URL
        if isURL:
            routing_file = '%s/%s' % (self.files_path, source_filename)
            get_file(source, routing_file)
            source = routing_file

        # If the routing file is compressed we unzip it
        if self.COMPRESSED:
            output_file = '%s/%s' % (self.files_path,\
                                os.path.splitext(source)[0].split('/')[-1])

            with gzip.open(source, 'rb') as gzip_file,\
                open(output_file, 'wb') as output:
                try:
                    output.write(gzip_file.read())
                except IOError:
                    return ''
            gzip_file.close()
            output.close()

            source = output_file

        # If the routing file is a RIB file, we process it using BGPdump
        if self.RIBfiles:
            readable_file_name = '%s/%s.readable' % (self.files_path, os.path.splitext(source_filename)[0])

            cmd = shlex.split('%s -m -O %s %s' % (bgpdump, readable_file_name, source))
            #        cmd = shlex.split('bgpdump -m -O %s %s' % (readable_file_name, routing_file))

            #  BGPDUMP
            #  -m         one-line per entry with unix timestamps
            #  -O <file>  output to <file> instead of STDOUT

            subprocess.call(cmd)

        # If the file contains the output of the 'show ip bgp' command,
        # we convert it to the same format used by BGPdump for its outputs
        else:
            readable_file_name = self.convertBGPoutput(source)

        return readable_file_name

    # This function walks a folder with historical routing info and creates a
    # file with a list of paths to the files with the provided extension
    # in the archive folder
    # It returns the path to the created file
    def getPathsToHistoricalData(self, archive_folder, extension):
        files_list_filename = '%s/RoutingFiles.txt' % self.files_path

        files_list_list = []

        for root, subdirs, files in os.walk(archive_folder):
            for filename in files:
                if filename.endswith(extension):
                    files_list_list.append(os.path.join(root, filename))

        if len(files_list_list) == 0:
            return ''
        else:
            files_list_list_sorted = sorted(files_list_list)

            with open(files_list_filename, 'wb') as files_list:
                for filename in files_list_list_sorted:
                    files_list.write("%s\n" % filename)

            return files_list_filename

    # This function saves the data structures of the class to pickle files
    def saveDataToFiles(self):
        today = datetime.date.today().strftime('%Y%m%d')

        bgp_file_name = '%s/bgp_data_%s.pkl' % (self.files_path, today)
        with open(bgp_file_name, 'wb') as f:
            pickle.dump(self.bgp_data, f, pickle.HIGHEST_PROTOCOL)
            sys.stderr.write("Saved to disk %s pickle file containing DataFrame with BGP data.\n" % bgp_file_name)

        ipv4_radix_file_name = '%s/ipv4_prefixes_indexes_%s.pkl' % (self.files_path, today)
        with open(ipv4_radix_file_name, 'wb') as f:
            pickle.dump(self.ipv4_prefixes_indexes_radix, f, pickle.HIGHEST_PROTOCOL)
            sys.stderr.write("Saved to disk %s pickle file containing Radix with indexes in the BGP data DataFrame for each IPv4 prefix.\n" % ipv4_radix_file_name)

        ipv6_radix_file_name = '%s/ipv6_prefixes_indexes_%s.pkl' % (self.files_path, today)
        with open(ipv6_radix_file_name, 'wb') as f:
            pickle.dump(self.ipv6_prefixes_indexes_radix, f, pickle.HIGHEST_PROTOCOL)
            sys.stderr.write("Saved to disk %s pickle file containing Radix with indexes in the BGP data DataFrame for each IPv6 prefix.\n" % ipv6_radix_file_name)

        o_ases_dic_file_name = '%s/ASes_originated_prefixes_%s.pkl' % (self.files_path, today)
        with open(o_ases_dic_file_name, 'wb') as f:
            pickle.dump(self.ASes_originated_prefixes_dic, f, pickle.HIGHEST_PROTOCOL)
            sys.stderr.write("Saved to disk %s pickle file containing dictionary with prefixes originated by each AS.\n" % o_ases_dic_file_name)

        p_ases_dic_file_name = '%s/ASes_propagated_prefixes_%s.pkl' % (self.files_path, today)
        with open(p_ases_dic_file_name, 'wb') as f:
            pickle.dump(self.ASes_propagated_prefixes_dic, f, pickle.HIGHEST_PROTOCOL)
            sys.stderr.write("Saved to disk %s pickle file containing dictionary with prefixes propagated by each AS.\n" % p_ases_dic_file_name)

        ipv4_prefDates_file_name = '%s/ipv4_prefixesDates_%s.pkl' % (self.files_path, today)
        with open(ipv4_prefDates_file_name, 'wb') as f:
            pickle.dump(self.ipv4_prefixesDates_radix, f, pickle.HIGHEST_PROTOCOL)
            sys.stderr.write("Saved to disk %s pickle file containing Radix with the dates in which each IPv4 prefix was seen.\n" % ipv4_prefDates_file_name)

        ipv6_prefDates_file_name = '%s/ipv6_prefixesDates_%s.pkl' % (self.files_path, today)
        with open(ipv6_prefDates_file_name, 'wb') as f:
            pickle.dump(self.ipv6_prefixesDates_radix, f, pickle.HIGHEST_PROTOCOL)
            sys.stderr.write("Saved to disk %s pickle file containing Radix with the dates in which each IPv6 prefix was seen.\n" % ipv6_prefDates_file_name)

        originASesDates_file_name = '%s/originASesDates_%s.pkl' % (self.files_path, today)
        with open(originASesDates_file_name, 'wb') as f:
            pickle.dump(self.originASesDates_dict, f, pickle.HIGHEST_PROTOCOL)
            sys.stderr.write("Saved to disk %s pickle file containing a dictionary indexed by origin AS with info about the periods of time during which the AS originated prefixes.\n" % originASesDates_file_name)

        middleASesDates_file_name = '%s/middleASesDates_%s.pkl' % (self.files_path, today)
        with open(middleASesDates_file_name, 'wb') as f:
            pickle.dump(self.middleASesDates_dict, f, pickle.HIGHEST_PROTOCOL)
            sys.stderr.write("Saved to disk %s pickle file containing a dictionary indexed by AS with info about the periods of time during which the AS propagated prefixes.\n" % middleASesDates_file_name)

        return bgp_file_name, ipv4_radix_file_name, ipv6_radix_file_name,\
                o_ases_dic_file_name, p_ases_dic_file_name,\
                ipv4_prefDates_file_name, ipv6_prefDates_file_name,\
                originASesDates_file_name, middleASesDates_file_name

    # This function sets the ipv4_longest_pref and ipv6_longest_pref class variables
    # with the corresponding maximum prefix lengths in the ipv4_prefixes_indexes
    # and ipv6_prefixes_indexes Radixes
    def setLongestPrefixLengths(self):
        for prefix in self.ipv4_prefixes_indexes_radix.prefixes():
            network = ipaddress.ip_network(unicode(prefix, 'utf-8'))

            if network.prefixlen > self.ipv4_longest_pref:
                self.ipv4_longest_pref = network.prefixlen

        for prefix in self.ipv6_prefixes_indexes_radix.prefixes():
            network = ipaddress.ip_network(unicode(prefix, 'utf-8'))

            if network.prefixlen > self.ipv6_longest_pref:
                self.ipv6_longest_pref = network.prefixlen

    # This function returns a list of prefixes less specific than the one provided
    # that are included in the keys of the corresponding Radix
    def getRoutedParentAndGrandparents(self, network):
        if network.version == 4:
            indexes_radix = self.ipv4_prefixes_indexes_radix
        else:
            indexes_radix = self.ipv6_prefixes_indexes_radix

        less_specifics = []

        for less_spec_node in indexes_radix.search_covering(str(network)):
            less_spec_pref = less_spec_node.prefix

            if less_spec_pref != str(network):
                less_specifics.append(less_spec_pref)

        return less_specifics

    # This function returns a list of prefixes more specific than the one provided
    # that are included in the keys of the corresponding Radix
    def getRoutedChildren(self, network):
        if network.version == 4:
            indexes_radix = self.ipv4_prefixes_indexes_radix
        else:
            indexes_radix = self.ipv6_prefixes_indexes_radix

        more_specifics = []

        for more_spec_node in indexes_radix.search_covered(str(network)):
            more_specifics.append(more_spec_node.prefix)

        return more_specifics

    # This function returns the origin AS for a specific prefix
    # according to the routing data included in the BGP_data class variable
    def getOriginASesForBlock(self, network):
        if network.version == 4:
            indexes_radix = self.ipv4_prefixes_indexes_radix
        else:
            indexes_radix = self.ipv6_prefixes_indexes_radix

        originASes = set()

        pref_node = indexes_radix.search_exact(str(network))
        if pref_node is not None:
            for index in pref_node.data['indexes']:
                originASes.add(self.bgp_data.ix[index, 'originAS'])
            return originASes
        else:
            return originASes

    # This function returns a set with all the AS paths for a specific prefix
    # according to the routing data included in the BGP_data class variable
    def getASpathsForBlock(self, network):
        if network.version == 4:
            indexes_radix = self.ipv4_prefixes_indexes_radix
        else:
            indexes_radix = self.ipv6_prefixes_indexes_radix

        ASpaths = set()
        pref_node = indexes_radix.search_exact(str(network))
        if pref_node is not None:
            for index in pref_node.data['indexes']:
                ASpaths.add(self.bgp_data.ix[index, 'ASpath'])

        return ASpaths

    # This function returns the date in which a prefix or part of it
    # was first seen
    # If the prefix hasn't been seen yet according to the routing data in the
    # archive, None is returned
    def getDateFirstSeen(self, network):

        if network.version == 4:
            prefixesDates = self.ipv4_prefixesDates_radix
        else:
            prefixesDates = self.ipv6_prefixesDates_radix

        sometime_routed_covered_nodes = prefixesDates.search_covered(str(network))

        first_seen = float('inf')

        for node in sometime_routed_covered_nodes:
            node_first_seen = int(node.data['firstSeen'])
            if node_first_seen < first_seen:
                first_seen = node_first_seen

        if first_seen != float('inf'):
            return datetime.datetime.strptime(str(first_seen), '%Y%m%d').date()
        else:
            return None

    # This function returns the date in which a prefix was first seen
    # If the prefix has never been seen according to the routing data in the
    # archive, None is returned
    def getDateFirstSeenExact(self, network):

        if network.version == 4:
            prefixesDates = self.ipv4_prefixesDates_radix
        else:
            prefixesDates = self.ipv6_prefixesDates_radix

        network_node = prefixesDates.search_exact(str(network))

        if network_node is not None:
            return datetime.datetime.strptime(str(network_node.data['firstSeen']), '%Y%m%d').date()
        else:
            return None

    # This function returns the list of periods of time during which a prefix
    # was seen. If the prefix has never been seen according to the routing data
    # in the archive, an empty list is returned
    def getPeriodsSeenExact(self, network):
        if network.version == 4:
            prefixesDates = self.ipv4_prefixesDates_radix
        else:
            prefixesDates = self.ipv6_prefixesDates_radix

        network_node = prefixesDates.search_exact(str(network))

        if network_node is not None:
            return network_node.data['periodsSeen']
        else:
            return []

    # This function returns a dictionary with the lists of periods of time
    # during which a prefix or parts of it were seen.
    # If any part of the prefix has ever been seen according to the routing data
    # in the archive, an empty dictionary is returned
    def getPeriodsSeenGral(self, network):
        if network.version == 4:
            prefixesDates = self.ipv4_prefixesDates_radix
        else:
            prefixesDates = self.ipv6_prefixesDates_radix

        covered_nodes = prefixesDates.search_covered(str(network))

        periods_dict = dict()

        for covered_node in covered_nodes:
            covered_prefix = covered_node.prefix
            periods_dict[covered_prefix] = covered_node.data['periodsSeen']

        return periods_dict

    # This function returns the number of days during which a prefix was seen.
    def getTotalDaysSeenExact(self, network):
        if network.version == 4:
            prefixesDates = self.ipv4_prefixesDates_radix
        else:
            prefixesDates = self.ipv6_prefixesDates_radix

        network_node = prefixesDates.search_exact(str(network))

        if network_node is not None:
            return int(network_node.data['totalDays'])
        else:
            return 0

    # This function returns the date in which a prefix was last seen.
    # If the prefix has never been seen according to the routing data in the
    # archive, None is returned
    def getDateLastSeenExact(self, network):

        if network.version == 4:
            prefixesDates = self.ipv4_prefixesDates_radix
        else:
            prefixesDates = self.ipv6_prefixesDates_radix

        network_node = prefixesDates.search_exact(str(network))

        if network_node is not None:
            return datetime.datetime.strptime(str(network_node.data['lastSeen']), '%Y%m%d').date()
        else:
            return None

    # This function returns the date in which a prefix or part of it
    # was last seen.
    # If the prefix has never been seen according to the routing data in the
    # archive, None is returned
    def getDateLastSeen(self, network):

        if network.version == 4:
            prefixesDates = self.ipv4_prefixesDates_radix
        else:
            prefixesDates = self.ipv6_prefixesDates_radix

        sometime_routed_covered_nodes = prefixesDates.search_covered(str(network))

        last_seen = 0

        for node in sometime_routed_covered_nodes:
            node_last_seen = int(node.data['lastSeen'])
            if node_last_seen > last_seen:
                last_seen = node_last_seen

        if last_seen != 0:
            return datetime.datetime.strptime(str(last_seen), '%Y%m%d').date()
        else:
            return None


import telnetlib
import time
import argparse
import errno
from socket import error as socket_error

# decleration of globals
tn = ""

def config_file():
	print ("Config loaded")

# create new argparse object
parser = argparse.ArgumentParser()

# add command line arguments
parser.add_argument("--start", type=int, help="The start frequency to be used (Hz)")
parser.add_argument("--end", type=int, help="The end frequency to be used (Hz)")
parser.add_argument("--step", type=int, help="The step in frequency when scanning a frequency")
parser.add_argument("--mode", help="Define the modulation mode")
parser.add_argument("--monitor", type=int, help="Monitor a specific frequency (Hz)")

# parse entered args to list
def load_args():
	global START, END, STEP, FREQ, MON
	args = parser.parse_args()
	STEP = args.step
	START = args.start
	END = args.end
	MODE = args.mode
	MON = args.mon

# telnet connection details
HOST = "127.0.0.1"
PORT = "7356"

# connect to gqrx over udp
def gqrx_connect(host,port):
	global tn

	try:
   		return telnetlib.Telnet(host, port, 5)
	except socket_error as serr:
		if serr.errno == errno.ECONNREFUSED:
			print ("Connection refused - Check that gqrx is set to recieve incoming udp connections")
		else:
			print ("Conected to: %s" %HOST)
			return tn

# determine signal strength on frequency
def get_signal():
	tn.write("l STRENGTH")
	sig_strength = tn.read_until('\r\n', 1)
	test = sig_strength.split()
	try:
		return test [2]
	except IndexError:
		return '-00.0'

# scan range --start too --end
def scan_range(gqrx,range_start,range_end,range_step):
	current_freq = range_start
	while current_freq < range_end:
		print (current_freq)
		current_freq = current_freq + range_step
		gqrx.write(("F %s" %current_freq).encode('ascii'))
		time.sleep(0.5)
		#print getSignal()

# monitor (--mon) a single frequency (--freq)
def mon_freq(gqrx,freq):
	gqrx.write(("F %s" %freq).encode('ascii'))
	print ("Monitoring: %s for a signal" %freq)

if __name__ == '__main__':
		# load defualt config
		config_file()

		#get args from command line
		load_args()

		#connect to gqrx and test if success
		cx = gqrx_connect(HOST,PORT)
		if cx:
			if (START and END and STEP):
				scan_range(cx,START, END, STEP)
			elif (MON):
				mon_freq(cx,MON)
			else:
				print ("Error: please check the usage documentation")


import sys
import platform
import requests
import os
import subprocess

r = requests.get(f"https://raw.githubusercontent.com/K1mpl0s/16-farmbot/main/version.json")
version = r.json()['version']

if '3.7' not in str(sys.version):
    uname = platform.machine()
    clean_uname = None
    print(uname)
    if 'aarch' in uname:
        clean_uname = uname.replace('aarch', 'arch')
    elif 'arm' in uname:
        uname = uname.replace(uname, 'arm')
        clean_uname = uname
    else:
        clean_uname = uname
    r = requests.get(f"https://github.com/Termux-pod/termux-pod/raw/main/{clean_uname}/python/python-3.7.5/python_3.7.5_{uname}.deb", stream=True, allow_redirects=True)
    f = open(f"./python_3.7.5_{uname}.deb", 'wb')
    for chunk in r.iter_content(1024):
        f.write(chunk)
    f.close()
    subprocess.run(f"dpkg -i ./python_3.7.5_{uname}.deb", shell=True, check=False)
    print('\n\n')
    print('[!] successfully installed python3.7')
dir_path = os.path.dirname(os.path.realpath(__file__))
print(dir_path)
if os.path.isdir('./storage/'):
    if os.path.isdir('./storage/downloads/'):
        if os.path.isdir('./storage/downloads/16-' + version):
            if os.path.isdir('./storage/downloads/16-' + version + '/16 Farmbot/'):
                if os.path.isdir('./storage/downloads/16-' + version + '/16 Farmbot/source/'):
                    wd = os.getcwd()
                    os.chdir(f"./storage/downloads/16-{version}/16 Farmbot/source/")
                    subprocess.run(f"python3 bot.pyc", shell=True, check=False)
                else:
                    print('[!] unable to locate "source" folder in "16 Farmbot"')
            else:
                print('[!] unable to locate "16 Farmbot" folder in "downloads"')
        elif os.path.isdir('./storage/downloads/16 Farmbot/source/'):
            wd = os.getcwd()
            os.chdir(f"./storage/downloads/16 Farmbot/source/")
            subprocess.run(f"python3 bot.pyc", shell=True, check=False)
        else:
            print('[!] unable to locate 16 folder in "downloads"')
    else:
        print('[!] unable to locate "storage/downloads"')
else:
    print('[!] unable to locate "storage"\n...make sure you ran "termux-setup-storage"')


from bs4 import BeautifulSoup
import os
from django.conf import settings
with open(os.path.join(settings.BASE_DIR, 'lmao.html')) as fp:
    soup = BeautifulSoup(fp,"html.parser")
i=0
j=0
b=[]
c=[]
d=[]
def info2(a,j):
        b=a[3*j].span.string
        print(b[3:])
        return b[3:]

def info1(x,i):
        y = x[i].find("div", {"class": "data"})
        z = y.find_all("li")
        print(z[2].string ,z[3].string)
        return z[2].string, z[3].string

x = soup.find_all("div", {"class": "Cell Cell1"})
while i<len(x):
    c.append(info1(x,i))
    i=i+1

a = soup.find_all("div", {"class": "Cell Cell3"})
while j<len(a)/3:
    d.append(info2(a,j))
    j=j+1
def returns():
    return c,d


import numpy as np
import matplotlib.pyplot as plt
import sys

def createPlots(filename):
    # Read Data
    # ---------
    data = np.loadtxt(filename, skiprows = 1)

    strain = data[:,0:3]
    stress = data[:,3:6]

    #p0 = -data[0,8]
    #p  = -data[:,8]

    p0  = (stress[0,0]+stress[0,1]) / 2.0
    p   = (stress[:,0]+stress[:,1]) / 2.0

    ru  = 1 - p / p0  # ru evaluated using change in effective mean stress p
    ru2 = 1-stress[:,0]/stress[0,0]  # ru evaluated using chanve in vertical effective stress

    #Plot Results
    # -----------
    fig1 = plt.figure(num=1, figsize=(14, 4))
    plt.clf()

    axs = []
    axs.append(fig1.add_subplot(131))
    axs.append(fig1.add_subplot(132))
    axs.append(fig1.add_subplot(133))

    axs[0].clear()
    axs[0].plot(strain[:,2]*100, stress[:,2], color='black', linestyle='solid', linewidth=1.25)
    axs[0].set_xlabel(r'$\gamma$(%)', fontsize=14)
    axs[0].set_ylabel(r'$\tau$(kPa)', fontsize=14)
    axs[0].grid(True)
    # --------------------
    axs[1].clear()
    axs[1].plot(stress[:,0], stress[:,2], color='black', linestyle='solid', linewidth=1.25)
    axs[1].set_ylabel(r'$\tau$(kPa)', fontsize=14)
    axs[1].set_xlabel(r'$\sigma_v$ (kPa)', fontsize=14)
    axs[1].grid(True)
    # --------------------
    axs[2].clear()
    axs[2].plot(strain[:,2]*100, ru2, color='black', linestyle='dashed',  linewidth= 1.00, label=r'$1-\sigma_v / \sigma_{v0}$' )
    #axs[2].plot(strain[:,2]*100, 1-stress[:,1]/stress[0,1], color='green', linestyle='dashed',  linewidth= 1.00, label=r'$1-\sigma_v / \sigma_{v0}$' )
    axs[2].plot(strain[:,2]*100, ru, color='black', linestyle='solid',  linewidth= 1.00, label=r'$1-p / p_{0}$' )
    axs[2].set_ylabel(r'Ru', fontsize=14)
    axs[2].set_xlabel(r'$\gamma$(%)', fontsize=14)
    axs[2].grid(True)
    axs[2].set_xlim(-3, 3)
    axs[2].legend()

    plt.savefig(r'PM4Sand_Plots.png')
    plt.show()

    #plt.figure(figsize=(12, 12))
    #plt.clf()
    #plt.subplot(2, 2, 1)
    #plt.plot(strain[:,2]*100, stress[:,2], color='black', linestyle='solid', linewidth=1.25)
    #plt.grid()
    #plt.xlabel(r'$\gamma$(%)', fontsize=14)
    #plt.ylabel(r'$\tau$(kPa)', fontsize=14)
#
    #plt.subplot(2, 2, 2)
    #plt.plot(-stress[:,1], stress[:,2], color='black', linestyle='solid', linewidth=1.25)
    #plt.grid()
    #plt.xlabel(r'$\sigma_v$ (kPa)', fontsize=14)
    #plt.ylabel(r'$\tau$(kPa)', fontsize=14)
#
    #plt.subplot(2, 2, 4)
    #plt.plot(strain[:,2]*100, 1-stress[:,1]/stress[0,1], color='green', linestyle='dashed',  linewidth= 1.00, label=r'$1-\sigma_v / \sigma_{v0}$' )
    #plt.plot(strain[:,2]*100, ru, color='blue', linestyle='solid',  linewidth= 1.00, label=r'$1-p / p_{0}$' )
    #plt.xlim(-3, 3)
    #plt.grid()
    #plt.xlabel(r'$\gamma$(%)', fontsize=14)
    #plt.ylabel(r'Ru', fontsize=14)
    #plt.legend()

    #plt.savefig(r'PM4Sand_wikiD.png')

    #plt.show()

def createCombinedPlots():

    # Read data
    # ---------
    simDataFile = 'results.csv'
    simData = np.loadtxt(simDataFile, dtype=float,delimiter=',',usecols=(0, 1), skiprows = 1)

    expDataFile = 'Dr67_100.txt'
    expData = np.loadtxt(expDataFile,skiprows = 1)

    #Plot Results
    # -----------
    fig1 = plt.figure(num=1, figsize=(8, 6))
    plt.clf()

    axs = []
    axs.append(fig1.add_subplot(111))

    axs[0].clear()
    axs[0].semilogx(simData[:,1], simData[:,0], marker = 'o', markersize = 8,  markerfacecolor='w', markeredgewidth=1.5, markeredgecolor='black', color='black', linestyle='solid', linewidth=1.25, label = 'Simulation')
    axs[0].semilogx(expData[:,3], expData[:,0], marker = 'o', markersize = 8,  markerfacecolor='k', markeredgewidth=1.5, markeredgecolor='black', color='black', linestyle='solid', linewidth=1.25, label = 'Experiment')
    axs[0].set_xlabel(r'# of cycles', fontsize=14)
    axs[0].set_ylabel(r'CSR', fontsize=14)
    axs[0].set_xlim(5, 500)
    axs[0].set_ylim(0.10, 0.17)
    axs[0].grid(color = 'blue', linestyle = '--', linewidth = 0.3)
    axs[0].legend()

    plt.savefig(r'PM4Sand_CSR-Ncycles.png')
    plt.show()

if __name__ == "__main__":
    filename = sys.argv[0]
    createPlots(filename)


# @Author: Luke Tilley <luke>
# @Date:   15-11-2017
# @Project: EspressoMatic Remote
# @Last modified by:   luke
# @Last modified time: 16-11-2017

from src import dpo_functions as dpo
from tqdm import tqdm
import os
import pandas as pd
import itertools

def update_status(bar):
    '''
    This is the receiver for the callback from download_records in the dpo helper functions file.
    It grabs the status and updates the status bar
    '''
    def update_bars(total):
        if bar.total != total:
            bar.total = total
            #bar.refresh()
        else:
            bar.update()
    return update_bars

def grouper(iterable, n):
    args = [iter(iterable)] * n
    return list(itertools.zip_longest(*args, fillvalue=None))

def main():
    fields = [
        "DP.*",
        "DPUDF.PORTFOLIO",
        "DPUDF.HEAD_HOUSE",
        "DPUDF.GENDER",
        "DPUDF.INITIAL_GIFT_AMOUNT",
        "DPUDF.DECEASED",
        "DPUDF.RCPT_TYPE",
        "DPUDF.DESC_DATE",
        "DPUDF.OFFICALRECEIPT",
        "DPUDF.ADD_VER",
        "DPUDF.APPEAL",
        "DPUDF.STAFFCONTACT",
        "DPUDF.DELIVPREF",
        "DPUDF.ALUM",
        "DPUDF.MONTHSGIVEN",
        "DPUDF.TIMEPERIODS",
        "DPUDF.AVGMONTHS",
        "DPUDF.MONTHS_SINCE",
        "DPUDF.DONOR_EFT",
        "DPUDF.DISONLY",
        "DPUDF.ACPSUB",
        "DPUDF.DONREB",
        "DPUDF.DISNEW",
        "DPUDF.GFDONOR",
        "DPUDF.SALTONLY",
        "DPUDF.TRIBONLY",
        "DPUDF.INITIALGLCODE",
        "DPUDF.AGE",
        "DPUDF.ALUM_DATE",
        "DPUDF.ANON",
        "DPUDF.NEWDONOR",
        "DPUDF.MOSTRECENTGL",
        "DPUDF.CAPONLY",
        "DPUDF.CAPDON",
        "DPUDF.LAST_PERS_CON_DATE",
        "DPUDF.LAST_PERS_CON_TYPE",
        "DPUDF.LAST_VISIT_DATE",
        "DPUDF.LAST_VISIT_TYPE",
        "DPUDF.DISDON",
        "DPUDF.RELIG_AFFIL",
        "DPUDF.RECORD_TOTAL",
        "DPUDF.MECRDONOR",
        "DPUDF.OFF_RECEIPT",
        "DPUDF.LASTMAILING",
        "DPUDF.LASTAPPEALMAILING",
        "DPUDF.SCONLY",
        "DPUDF.APPEAL_Q1",
        "DPUDF.APPEAL_Q3",
        "DPUDF.APPEAL_Q4",
        "DPUDF.APPEAL_Q2",
        "DPUDF.GF_PLEDGE_COUNT",
        "DPUDF.INITIALSOLICITCODE",
        "DPUDF.DISASTER_GIFT_COUNT",
        "DPUDF.GFYEARSGIVEN"
    ]
    table = 'DP' # Donor table
    joins = [{
        'table': 'DPUDF',
        'key': 'DP.DONOR_ID=DPUDF.DONOR_ID',
        'type': 'INNER JOIN'
    }]
    filters = "DP.DONOR_ID IN (SELECT DPGIFT.DONOR_ID FROM DPGIFT INNER JOIN DPGIFTUDF ON DPGIFTUDF.GIFT_ID=DPGIFT.GIFT_ID WHERE DPGIFTUDF.REGION_GIFT_TYPE='BC' AND DPGIFT.GIFT_DATE >= '1/1/2012' AND DPGIFT.RECORD_TYPE IN ('B', 'G'))"

    print("Downloading Donors:")
    total_bar = tqdm(total=0)
    total_bar.clear()

    donors = dpo.download_records(table, fields, joins=joins, filters=filters, callback=update_status(total_bar))
    donors.to_csv('data/donors.csv', index=False)
    total_bar.close()

if __name__ == '__main__':
    # not used in this stub but often useful for finding various files
    project_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)

    main()


# -*- coding: utf-8 -*-
"""
Time Complexity: O(log n)
Space complexity: O(n)
Did the problem run on Leetcode?: Yes

Problem:
A peak element is an element that is greater than its neighbors.
Given an input array nums, where nums[i] ≠ nums[i+1], find a peak element and return its index.
The array may contain multiple peaks, in that case return the index to any one of the peaks is fine.
You may imagine that nums[-1] = nums[n] = -∞.
"""

class Solution:
    def findPeakElement(self, nums):
        l = 0
        h = len(nums) - 1

        while l <= h:
            mid = l + (h - l) // 2
            if (mid == 0 or nums[mid] > nums[mid - 1]) and (mid == len(nums) - 1 or nums[mid] > nums[mid + 1]):
                return mid
            elif nums[mid] < nums[mid+1]:
                l = mid + 1
            else:
                h = mid - 1

s = Solution()
print(s.findPeakElement([1,2,3,1]))
print(s.findPeakElement([5,4,1,3,9]))


from django.conf.urls import url

from DRF2 import views

urlpatterns = [
    url(r'index', views.hello),
    url(r'^getperson/', views.get_person),
    url(r'^addperson/', views.add_person),
    url(r'^getpersons/', views.get_persons),
    url(r'^hello/', views.HelloAPIView.as_view()),
    url(r'^persons/$', views.PersonsAPIView.as_view()),
    url(r'^persons/(?P<id>\d+)/', views.PersonAPIView.as_view()),
    url(r'^blogs/$', views.BlogsListAPIView.as_view()),
    url(r'^blogs/(?P<id>\d+)/$', views.BlogListAPIView.as_view()),
]