import torch
from torch import nn
import torchvision

class Encoder(nn.Module):
    """
    Encoder
    """
    def __init__(self, encoded_image_size=14):
        super(Encoder, self).__init__()
        self.enc_img_size = encoded_image_size

        cnn_ext = torchvision.models.resnet50(pretrained = True)  # 使用预训练的 resnet-50
        modules = list(cnn_ext.children())[:-2]  # 去掉网络中的最后两层，考虑使用 list(cnn_ext.children())
        self.cnn_ext = nn.Sequential(*modules)  # 使用 nn.Sequential 定义好 encoder

        self.adaptive_pool = nn.AdaptiveAvgPool2d((encoded_image_size, encoded_image_size))  # 使用 nn.AdaptiveAvgPool2d 将输出改变到指定的大小

    def forward(self, img):
        out = self.cnn_ext(img)  # [bs, 2048, 8, 8]
        out = self.adaptive_pool(out)  # [bs, 2048, enc_img_size, enc_img_size]
        out = out.permute(0, 2, 3, 1)  # [bs, enc_img_size, enc_img_size, 2048]
        return out

    def freeze_params(self, freeze):
        for p in self.cnn_ext.parameters():
            p.requires_grad = False

        for c in list(self.cnn_ext.children())[5:]:
            for p in c.parameters():
                p.requires_grad = (not freeze)

class AttentionModule(nn.Module):
    """
    Attention Module with Decoder
    """

    def __init__(self, encoder_dim, decoder_dim, attention_dim):
        """
        :param encoder_dim: 图片经过 Encoder 之后的特征维度
        :param decoder_dim: 解码器隐含状态 h 的维度
        :param attention_dim: attention的维度
        """
        super(AttentionModule, self).__init__()

        self.encoder_att = nn.Linear(encoder_dim, attention_dim)  # Linear, encoder_dim -> attention_dim
        self.decoder_att = nn.Linear(decoder_dim, attention_dim)  # Linear, decoder_dim -> attention_dim
        self.relu = nn.ReLU()  # relu 激活函数
        self.softmax = nn.Softmax(dim=1)  # softmax 激活函数, dim=1

    def forward():
        """
        注意力机制的前向传播过程
        待完成
        """

class Decoder(nn.Module):

    def __init__(self, embed_dim, decoder_dim, vocab_size, encoder_dim=2048, dropout=0.5):
        """
        需要加入attention机制（待完成）
        :params embed_dim: 词向量的维度
        :params decoder_dim: 解码器的维度
        :params vocab_size: 单词总数
        :params encoder_dim: 编码图像的特征维度
        :params dropout: dropout 的比例
        """
        super(Decoder, self).__init__()
        # 定义类中的参数
        self.encoder_dim = encoder_dim
        self.embed_dim = embed_dim
        self.decoder_dim = decoder_dim
        self.vocab_size = vocab_size

        # 定义decoder中需要的网络层
        self.embedding = nn.Embedding(vocab_size, embed_dim)  # 定义词嵌入 word embedding, (vocab_size, embed_dim)
        self.dropout = nn.Dropout(dropout)  # 定义 dropout，dropout = 0.5
        self.decode_step = nn.LSTMCell(embed_dim + encoder_dim, decoder_dim,
                                       bias=True)  # 定义 LSTMCell 作为 Decoder 中的序列模块，输入是 embed + encoder_out
        self.init_h = nn.Linear(encoder_dim, decoder_dim)  # 定义线性层将 encoder_out 转换成 hidden state
        self.init_c = nn.Linear(encoder_dim, decoder_dim)  # 定义线性层将 encoder_out 转换成 cell state
        self.f_beta = nn.Linear(decoder_dim, encoder_dim)  # 定义线性层, decoder_dim -> encoder_dim
        self.sigmoid = nn.Sigmoid()  # 定义 sigoid 激活函数F
        self.fc = nn.Linear(decoder_dim, vocab_size)  # 定义输出的线性层

    def init_hidden_state(self, encoder_out):
        """
        给 LSTM 传入初始的 hidden state，其依赖于 Encoder 的输出
        :param encoder_out: 通过 Encoder 之后的特征，维度是 (bs, num_pixels, encoder_dim)
        :return: hidden state, cell state
        """
        # 对所有的像素求平均
        mean_encoder_out = encoder_out.mean(dim=1)
        # 线性映射分别得到 hidden state 和 cell state
        h = self.init_h(mean_encoder_out)
        c = self.init_c(mean_encoder_out)
        return h, c

    def forward(self, encoder_out, encoded_captions, caption_lens):
        """
        Decoder的向前传播机制
        :param:encoder_out:编码之后的特征，维度是(bs, num_pixels, encoder_dim)
        :param:encoder_caption:字幕，维度是(bs,max_caption_len)
        :param:caption_lens:字幕真正长度，维度是 (bs, 1)

        Returns:predictions，预测的字幕
        """
        batch_size = encoder_out.shape[0]
        encoder_dim = encoder_out.shape[-1]
        vocab_size = self.vocab_size

        # flatten encode_out 特征
        encoder_out = encoder_out.view(batch_size, -1,
                                       encoder_dim)  # (bs, size_pic,size_pic,encodet_dim)-->(bs, num_pixels, encoder_dim)
        num_pixels = encoder_out.size(1)

        # 对输入的字幕长度按照降序排列,同时对encoder_out也按照该顺序进行排列
        # 对应原理：分批次训练时，字幕长度短的训练次数少
        caption_lens, sort_idx = caption_lens.squeeze(1).sort(dim=0, descending=True)
        encoder_out = encoder_out[sort_idx]
        encoded_captions = encoded_captions[sort_idx]
        embeddings = self.embedding(
            encoded_captions)  # 得到 encoded_captions 的词向量, （bs, max_caption_len）-->(bs, max_caption_lens, embed_dim)

        # 初始化 LSTM hidden state 和 cell state
        h, c = self.init_hidden_state(encoder_out)

        # 不会对 <end> 位置进行解码，所以解码的长度是 caption_lens - 1
        decode_lens = (caption_lens - 1).tolist()

        # 定义存储预测结果的空 tensor
        predictions = torch.zeros(batch_size, max(decode_lens), vocab_size)

        # 在每个时间步，通过 decoder 上一步的 hidden state 来生成新的单词
        for t in range(max(decode_lens)):
            # 决定当前时间步的 batch_size，通过 [:batch_size_t] 可以得到当前需要的 tensor
            # t ==0; batch_size_t = all;  意味着长度越短的字幕迭代的次数越少
            batch_size_t = sum([l > t for l in decode_lens])
            print("t:", t, " bs_t:", batch_size_t);

            # 前向传播一个时间步，输入是 embeddings
            # hidden state 和 cell state 也需要输入到网络中，注意使用 batch_size_t 取得当前有效的 tensor
            h, c = self.decode_step(embeddings[:batch_size_t, t,:],
                (h[:batch_size_t], c[:batch_size_t]))

            preds = self.fc(self.dropout(h))  # 对 h 进行 dropout 和全连接层得到预测结果

            predictions[:batch_size_t, t, :] = preds

        return predictions, encoded_captions, decode_lens, sort_idx


#!/usr/bin/env python

from __future__ import print_function
import sys
import socket
import time

def get_key():
	try:
		sock1 = socket.socket()
		sock1.connect((server, int(port)))
		banner = sock1.recv(2048)
		sock1.send('getkey\n')
		key = sock1.recv(2048).rstrip()
		key = key.replace('Your key is: ', '')
		sock1.close()
	except:
		return -1

	return key

def try_char(c, key):
	try:
		time.sleep(2)
		sock2 = socket.socket()
		sock2.connect((server, int(port)))
		banner = sock2.recv(2048)
		sock2.send('givechar %s %s\n' % (c, key))
		pos = sock2.recv(2048).rstrip()
		sock2.close()
		if 'such sad' in pos:
			raise
	except:
		return -1

	return pos.split(',')

if not len(sys.argv) == 3:
    sys.exit(-1)

server = sys.argv[1]
port = sys.argv[2]

key = get_key()
key_dict = {}

for ord_x in range(33,127):
	key_items = try_char(chr(ord_x), key)
	if key_items != -1:
		key_dict[chr(ord_x)] = key_items

flag_len = int(key_dict['}'][0]) + 1
flag = [''] * flag_len

for letter, positions in key_dict.items():
	for pos in positions:
		flag[int(pos)] = letter

print(''.join(flag))


from pyramid.config import Configurator
from sqlalchemy import engine_from_config
from os import environ
from .models import get_counter_tables

def main(global_config, **settings):
    sqlalchemy_url_value = environ.get('MARIADB_STRING_CONNECTION', 'mysql://root:pass@172.17.0.3:3306/matomo')
    settings.update({'sqlalchemy.url': sqlalchemy_url_value})

    application_url_value = environ.get('APPLICATION_URL', 'http://127.0.0.1:6543')
    settings.update({'application.url': application_url_value})

    config = Configurator(settings=settings)

    engine = engine_from_config(settings, 'sqlalchemy.')
    counter_tables = get_counter_tables(engine)

    settings = config.registry.settings
    settings['counter_tables'] = counter_tables

    config.add_route('home', '/')
    config.add_route('status', '/status')
    config.add_route('members', '/members')
    config.add_route('reports', '/reports')
    config.add_route('reports_report_id', '/reports/{report_id}')
    config.scan('.views')

    return config.make_wsgi_app()


# Import the necessary modules
import pandas as pd  # for handling data
import requests # for pulling pages from the internet
from bs4 import BeautifulSoup  # for scraping and processing pages
import numpy as np # for statistical calculations
import matplotlib.pyplot as plt # for plotting histograms
from matplotlib.pyplot import figure # For plotting feature importance
from pandas.plotting import scatter_matrix # for scatter matrices

import sklearn
from sklearn.preprocessing import StandardScaler # For performing z-normalizations
from sklearn.model_selection import train_test_split # For splitting data
from sklearn.tree import DecisionTreeClassifier # Decision tree classifier
from sklearn.ensemble import RandomForestClassifier # Random forest classifier
from sklearn import svm # for Support Vector Machine classifier
from sklearn.metrics import accuracy_score # for generating accuracy scores

from sklearn.metrics import * # for model evaluation

#################  EXTRACT DATA AND PROCESS COLUMN HEADERS ####################

# Read the data from the website into a Pandas DataFrame.
df = pd.read_csv("https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data")

# Pull the page with the column names and metadata.
cols_url = requests.get("https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.names")

# Use BS4 to split the page into a list of strings
soup = BeautifulSoup(cols_url.content, "lxml").text.split('\n')

# Inspecting the HTML of the page reveals the headers are in dict format on lines 96-110
# Pull the lines with column strings from the HTML into a list.
col_strings = soup[96:110]

# Create function to the lines and rebuild them into a dictionary with column names and possible values.
def strings_to_dicts(s_list):
    col_dict = {}
    for item in s_list:
        entry = item.split(':')
        item = entry[0]
        key = entry[1]
        col_dict[item] = key
    return col_dict

# Run html stripping function.
colsd = strings_to_dicts(col_strings)

# Extract column names and convert to list format.
cols = list(colsd.keys())

# Put together missing column names.
cols = cols + ['income_cat']

# Provide the column names for the DataFrame using the column list.
df.columns = cols

############  CREATE INFO FUNCTION AND PERFORM DATA EXPLORATION #########

# Define function to get information from the DataFrame.
def df_info(df, dv = None):
    """Automatically combine dataset information into one dataframe.
       Use both pandas and numpy methods of getting
       max and min to help catch bad cell values."""

    # Create empty DataFrame for info to go in.
    info= pd.DataFrame()

    # Insert the datatypes.
    info['dtype'] = df.dtypes

    # Compute number of rows with one or more fields mising
    missing_rows_ct = df.shape[0] - df.dropna().shape[0]

    # Count non-null values in each column
    info['#values'] = df.count()

    # Count the null values and and insert them.
    info['is_null'] = df.isnull().sum()

    # Caclulate the percentage of the field is missing.
    info['pct_null'] = info['is_null'] / len(df)

    # Proportion of total rows missing
    info['pct_null_rows'] = info['is_null'] / missing_rows_ct

    # Count the duplicates and insert them.
    dupcount = df.duplicated().sum()

    # Run describe function and then transpose the values.
    stats = df.describe().transpose()

    # Transpose the stats DataFrame and then join it with the info DataFrame.
    info = info.join(stats,how='outer',rsuffix='_np')

    # Create blank dictionary for categorical column information.
    cat_info_dict = {}

    # For each column in the dataframe, check if datatype is object. If it is,
    # get the unique values, count them, and store them in the dictionary.
    for col in df:
        if df[col].dtype == 'object':
            column_items = df[col].unique()
            cat_info_dict.update({col : {'unique_count': len(df[col].unique()),
                                         'unique_items': column_items}})
        else:
            pass

    # Create DataFrame from dictionary.
    cat_df = pd.DataFrame.from_dict(cat_info_dict, orient= 'index')

    # Merge DataFrame with categorical information with main DataFrame.
    info = info.join(cat_df,how='outer')

    # Drop useless count column
    info = info.drop(['count'], axis=1)

    # Print the dataset information.
    print('DATASET INFORMATION:')
    print('{0} rows x {1} columns'.format(len(df),len(df.columns)))
    print('--> {0} rows have missing values'.format(missing_rows_ct))
    print('--> {0} rows are duplicates'.format(dupcount))

    # If a dependent variable is provided, provide measurements against that variable.
    if dv != None:
        # If the dv is categorical, get percentages matching each category.
        if df[dv].dtype == 'object':

            print('--> Dependent Variable Values:')

            # get count of unique values in dependent variable column
            cat_list = df[dv].unique()

            # Get count of the number of unique items.
            unique_items_count = len(cat_list)

            # Calculate prpoportion for each unique dependent variable value.
            for i in range(0, unique_items_count):
                # Use i to pull each item from list of items.
                category = cat_list[i]

                # Get number of items that match that value of the dependent variable
                category_count = len(df[(df[dv] == category)])

                # For each category the percent of values where the column value matches that category.
                category_prop = category_count / len(df)

                print('    {0} = {1} : {2}'.format(dv, category, category_prop))

        # I will add code for numerical dependent variables later if the need arises.
        else:
            pass

    return info

########### CLEAN DATA, REMOVE WHITESPACE AND REMOVE UNNECESSARY COLUMNS ###############

# Remove whitespace everywhere in df.
def remove_whitespace(df):
    """Remove all whitespace from all cells and columns in the DataFrame."""
    df_obj = df.select_dtypes(['object'])
    df[df_obj.columns] = df_obj.apply(lambda x: x.str.strip())
    return df

df = remove_whitespace(df)

# Since ' ?' is being used as a placeholder, let's remove entries with ?s in them.
# Thank you stack overflow for helping with me with this line (see citation at the bottom of script)
df = df[(df != '?').all(axis=1)]

# education-num is NOT referring to number of years of schooling - it is just a number code for education column.
# Since the education column already has these values decoded, let's drop the education-num column.
df = df.drop(['education-num'], axis=1)

# Let's look at the scatter matrix first to get a sense of the big picture.
scatter_matrix(df,figsize= (10,10))

# Convert DataFrame to json.
outfile = df.to_json(path_or_buf=None, orient='records', index=True)

# Save the raw dataset to disk.
with open('adult_raw.json', 'w') as f:
    f.write(outfile)

############ LOAD DATA ################################################
# Load dataset from disk into DataFrame.
df = pd.read_json('adult_raw.json', orient='columns'
                 'records', convert_axes=True, convert_dates=True, keep_default_dates=True,
                 numpy=False, precise_float=False, date_unit=None, encoding=None)

###########  REMOVE OUTLIERS AND REVIEW DISTRIBUTIONS #######################

# See what dataset looks like with question marks and whitespace removed.
df_info(df)

# Education-num, and hours per week both look normally distributed. There are a few outliers skewing the age distribution that may also
# be concealing a normal distrubtion. Lets remove them and see what happens.

def replace_outliers(df, field, placeholder='median'):
    """Remove the outliers from a coulumn in a dataframe, using the instructions provided
       in the dictionary.
       * Note that the data atrophy that can add up if you use this function multiple times."""

    # Start values replaced counter.
    replaced_count = 0

    # Get initial length of datset before processing.
    initial_length = len(df)

     # Extract the numeric field from the dataframe, getting the column name from the dict.
    series = df[field]

    # Get upper limit by adding the mean + 2 standard deviations.
    upper_limit = np.mean(series) + 2*np.std(series)

    # Get the lower limit by subtracting two standard deviations from the mean.
    lower_limit = np.mean(series) - 2*np.std(series)

    # Flag entries with value above upper_limit and count them.
    high_outliers = df.loc[:, field] > upper_limit

    # Flag low outliers with values below lower_limit and count them.
    low_outliers = df.loc[:, field] < lower_limit

    # Valid entries is the complement of outliers
    valid_entries = ~(high_outliers | low_outliers)

    # If the user specifies to use the median, calculate the placeholder value from the median.
    if placeholder == 'median':
        repl = np.median(series)

    # If the user specifies to use the mean, use the valid entries to calculate the mean for the placeholder.
    if placeholder == 'mean':
        repl = np.mean(df.loc[valid_entries, field])

    # Replace both high and low outliers with placeholder value.
    replaced_count += df.loc[high_outliers, field].count()
    df.loc[high_outliers, field] = repl

    replaced_count += df.loc[low_outliers, field].count()
    df.loc[low_outliers, field] = repl

    replaced_portion = replaced_count/initial_length
    print('{0} outliers removed ({1})'.format(replaced_count, replaced_portion))

    # Return the DataFrame with the outliers removed.
    return df

# Run function to remove outliers from age field.
df = replace_outliers(df, 'age')

# Remove outliers from census weights field and see what happens
df = replace_outliers(df, 'fnlwgt')

plt.hist(df['fnlwgt'])
# Looks like with outliers removed, census weights are normally distributed. The data loss
# is a little less than 4% of the remaining data
plt.show()

scatter_matrix(df,figsize= (10,10))

########### CREATE Z SCALER FUNCTION (WILL RUN LATER) ################
def batch_zscaler(data, column_list):
    """ Run SKL Standard Scaler to z normalize a list of columns.
        Inputs:
        - data (a dataframe)
        - column_list (a list of strings specifying the columns to normalize)"""
    for col in column_list:
        # Now that the outliers are removed from the age column, the age column looks normally distributed.
        # Let's make a z-score column for the age attribute.
        col_values = data[col]

        # Reshape the column values array
        col_values_reshaped = col_values.values.reshape(-1,1)

        # Fit z scaler according to values in column.
        standardization_scale = StandardScaler().fit(col_values_reshaped)

        # Transform values according to z-scaler
        data[col] = standardization_scale.transform(col_values_reshaped)

    return data

########### STEP 6: BIN HOURS PER WEEK INTO CATEGORICAL VARIABLES ##################

def bin_numeric(x, bin_count): # data = data array, bounds = boundaries array
    """Take an array with the data and an array with the bin boundaries
    and then bin the data according to the boundaries supplied"""

    # Get bounds of data using minimum and maximum values of data and bin count.
    bounds = np.linspace(np.min(x), np.max(x), bin_count + 1)

    # Get number of bins
    bounds_count = len(bounds)

    # Get length of the array to be binned.
    array_length = len(x)

    # Create empty integer array to store the bin numbers (output)
    y = np.empty(array_length, int)

    # For each boundary, tag the data within each boundary, iterating
    # until all boundaries are tagged.
    for i in range(1, bounds_count):
        # If the data is greater than the bound of bin i-1 and less
        # than the upper bound of bin i, set the bin to i
        y[(x >= bounds[i-1]) & (x < bounds[i])] = i

    # If the value of x is right on the borderline of one of the bounds, bin it in the bound below it.
    y[x == bounds[-1]] = bounds_count - 1

    # Return the array of the bounds correspoding to each value in the array.
    return y

# Pull the data from the column into a variable.
hours = df['hours-per-week']
hours_vals = hours.values

# The hours per week range from 0 - 100. Let's bin it into 9 bins.
x_binned = bin_numeric(hours_vals, 9)

# Store the hours category in a new column in the DataFrame.
df['hours_cat'] = x_binned

# Rename categories to reflect approximate boundaries. Note that these boundaries are not exact since the boundaries are not exact integers.
df.loc[df.loc[:, "hours_cat"] == 1, "hours_cat"] = "works-0-11h"
df.loc[df.loc[:, "hours_cat"] == 2, "hours_cat"] = "works-12-22h"
df.loc[df.loc[:, "hours_cat"] == 3, "hours_cat"] = "works-23-34h"
df.loc[df.loc[:, "hours_cat"] == 4, "hours_cat"] = "works-34-45h"
df.loc[df.loc[:, "hours_cat"] == 5, "hours_cat"] = "works-45-55h"
df.loc[df.loc[:, "hours_cat"] == 6, "hours_cat"] = "works-55-66h"
df.loc[df.loc[:, "hours_cat"] == 7, "hours_cat"] = "works-66-77h"
df.loc[df.loc[:, "hours_cat"] == 8, "hours_cat"] = "works-77-88h"
df.loc[df.loc[:, "hours_cat"] == 9, "hours_cat"] = "works-88-100h"

############ STEP 7: PROCESS ADDITIONAL CATEGORICAL VARIABLES AND CREATE DUMMY VARIABLES

# Judging from the info output, many of the missing fields in workclass are also missing occupation fields.
# Since I anticipate these fields are going to be very important for the model, let's drop the missing entries in workclass
df = df[pd.notnull(df['occupation'])]

# Looks like removing this field removed all but about 500 the missing entries, which is convenient.
# Since we have a dense dataset, let's take a closer look at these missing entries.
missing = df[pd.isnull(df['native-country'])]

# With an excption of some outliers in the capital-gains and capital-losses attributes, these values appear to be similarly distributed
# to the population. These account for around 2% of the dataset.

# If I worked at the census, I would ask a domain expert what their
# sense of who these people might be. Let's say I was told they are
# mostly Americans.
df.loc[df.loc[:, "native-country"].isna(), "native-country"] = ' United-States'

# Create column categorizing whether indidivual is employed by the government.
govt_cats = ["Federal-gov", 'Local-gov', 'State-gov']
df.loc[df['workclass'].isin(govt_cats), 'is_govt_emp'] = 1
df.loc[~df['workclass'].isin(govt_cats), 'is_govt_emp'] = 0

# Create column categorizing whether indidivual is self-employed.
self_emp_cats = ['Self-emp-not-inc', 'Self-emp-inc']
df.loc[df['workclass'].isin(self_emp_cats), 'self_emp'] = 1
df.loc[~df['workclass'].isin(self_emp_cats), 'self_emp'] = 0

# Since there are tons of different values for native country, this might slow down the algorhythum, so lets encode the variable
# into a boolean column for whether the individal is an immigrant.
df["is-immigrant"] = (df['native-country'] != " United-States").astype(int)

# Let's drop country of origin. Since we have a saved copy we can bring it back later if necessary.
df = df.drop(['native-country'],axis=1)

# View the dataset after changes were made.
info = df_info(df, 'income_cat')
print(info)

# Select columns to encode with dummies.
# Format is 'column_name' : drop?
dummycols = {'sex' : True,
               'income_cat': True,
                'hours_cat' : True,
                'education' : True,
                'marital-status' : True,
                'occupation' : True,
                'race' : True,
                'relationship' : True,
                'workclass' : True}

def make_dummies(df, colinfo):
    """Use instructions from user-supplied dictionary to create category
       dummy columns in a dataframe."""
    for key, value in colinfo.items():

        # Get column name from dict.
        colname = key

        # Get boolean instruction on whether to drop obsolete columns from dict.
        drop = value

        # Get list of unique items in the selected column.
        unique_items = df[colname].unique()

        # Get count of the number of unique items.
        unique_items_count = len(df[colname].unique())

        # For all but 1 of the unique items,
        for i in range(1, unique_items_count):
            # Use i to pull each item from list of items.
            item = unique_items[i]

            # Remove spaces from item string so that column name is neater.
            item.strip()

            # Generate column name from item name and build column with boolean value.
            df.loc[:, "is_{0}".format(item)] = (df.loc[:, colname] == item).astype(int)

        # If user wants to drop the obsolete column, drop it.
        if drop == True:
            df = df.drop([colname],axis=1)

        # If user does not want to drop obsolete column, pass.
        if drop == False:
            pass

    # Return modified DataFrame
    return df

df2 = make_dummies(df, dummycols)

wk = df['workclass'].values
print(wk)

# Let's see what the DataFrame looks like now.
print(df2[:5])

info2 = df_info(df2)

print(df2.columns)

## SAVE DATASET TO DISK
df2.to_csv('Berkowitz-M02-Dataset.csv', header = True)

################ DEFINE HELPER FUNCTIONS FOR RESULTS ANALYSIS ###############

def feature_importance(model, X_train):
    """Plot importances for each feature in a given model.
    Inputs: Model and Training Data
    Outputs: Plotted feature importances"""

    # Pull feature labels from the training data.
    feat_labels = X_train.columns

    # Pull importances array from the selected model object.
    importances = model.feature_importances_

    ## PLOT IMPORTANCES

    # Sort the importances, largest to smallest.
    indices = np.argsort(importances)[::-1]

    # Set figure size.
    figure(num=None, figsize=(15, 6), dpi=80, facecolor='w', edgecolor='k')

    # Set plot title.
    plt.title('Feature Importance')

    # Set bar plot.
    plt.bar(range(X_train.shape[1]),
            importances[indices],
            align='center')

    # Set X ticks.
    plt.xticks(range(X_train.shape[1]),
               feat_labels[indices], rotation=90)

    # Set X limit.
    plt.xlim([-1, X_train.shape[1]])

    # Show plot.
    plt.show()

def make_confmat(y_true, y_pred):
    """Calculate and graph a confusion matrix based on arrays with
       true and predicted values."""
    confmat = confusion_matrix(y_true=y_true, y_pred=y_pred)

    fig, ax = plt.subplots(figsize=(2.5, 2.5))
    ax.matshow(confmat, cmap=plt.cm.Blues, alpha=0.3)
    for i in range(confmat.shape[0]):
        for j in range(confmat.shape[1]):
            ax.text(x=j, y=i, s=confmat[i, j], va='center', ha='center')

    plt.xlabel('Predicted label')
    plt.ylabel('True label')

    plt.tight_layout()
    #plt.savefig('images/06_09.png', dpi=300)
    plt.show()

    # Show detailed statistics

    # Generate accuracy score from arrays of target values and actual values.
    AR = accuracy_score(y_true, y_pred)
    print("\nAccuracy rate:", AR)

    # Calculate error rate by subtracting accuracy score from 1.
    ER = 1.0 - AR
    print("\nError rate:", ER)

    # Calculate precision score from target array and values array.
    P = precision_score(y_true, y_pred)
    print ("\nPrecision:", np.round(P, 2))

    # Calculate recall score from target array and values array.
    R = recall_score(y_true, y_pred)
    print ("\nRecall:", np.round(R, 2))

    # Calculate F1 score from target array and values array.
    F1 = f1_score(y_true, y_pred)
    print ("\nF1 score:", np.round(F1, 2))

def plot_auc(targets, predictions, LW = 1.5, LL = 'lower right', LC = 'darkgreen'):
    """Plot AUC to show model preformance.
    Inputs:
      targets = 1D Numpy array with prediction targets.
      predictions = 1D Numpy array with predictions.
      Variables customizing graph appearance (optional)
    Outputs:
      True Positive Rates (TPR)
      AUC calculation.
      Plot of the ROC curve.
      """
    LW = LW # line width for plots
    LL = LL # legend location
    LC = LC # Line Color

    fpr, tpr, th = roc_curve(targets, predictions) # False Positive Rate, True Posisive Rate, probability thresholds

    # Calculate AUC score.
    AUC = auc(fpr, tpr)

    # Display calculated scores, rounded to the second digit.
    print("\nTP rates:", np.round(tpr, 2))
    print("\nFP rates:", np.round(fpr, 2))
    print("\nProbability thresholds:", np.round(th, 2))

    # Generate figure for ROC curve.
    plt.figure()
    plt.title('Receiver Operating Characteristic curve')

    # Set limits to the x and y ranges to give the plot some breathing room.
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])

    plt.xlabel('FALSE Positive Rate')
    plt.ylabel('TRUE Positive Rate')

    # Plot False Positive Rate against True Positive Rate.
    plt.plot(fpr, tpr, color=LC,lw=LW, label='ROC curve (area = %0.2f)' % AUC)

    # Reference line for random classifier
    plt.plot([0, 1], [0, 1], color='navy', lw=LW, linestyle='--') # reference line for random classifier
    plt.legend(loc=LL)
    plt.show()

    # Show rounded AUC scores using auc and roc_auc functions.
    print("\nAUC score (using auc function):", np.round(AUC, 2))
    print("\nAUC score (using roc_auc_score function):", np.round(roc_auc_score(targets, predictions), 2), "\n")

###################################################################################

############ LOAD THE DATA ##############################
df = pd.read_csv('Berkowitz-M02-Dataset.csv')

print(df.columns)

############ SPLIT AND SCALE THE DATA ###################

## NOTE: I'm not going to normalize the age column since they have already been normalized.
# If I were to normalize them, however, heres the code I would use.
# # Transform values according to z-scaler and put them in new column.
# df['znorm-age'] = standardization_scale.transform(ages_reshaped)
####

# Use list to specify columns to ignore. I'm removing the unnamed column since that's just duplicated indexes and the is 50k column
# Since that is the target variable. Also removing hours-per-week since thats already been binned.
exclude_cols = ['is_>50K', 'Unnamed: 0', 'hours-per-week']

# Set y to the target variable.
y = df["is_>50K"]

# Set X to the dataset, with the excluded columns excluded.
X = df.loc[:, ~df.columns.isin(exclude_cols)]

# Extract features names from column titles.
features = X.columns

# Split data into training and test data using 40 percent for testing.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.40, random_state=42)

# Run the Z scaler on the training data.
X_train = batch_zscaler(X_train, ['age', 'fnlwgt'])

# Run the Z scaler on the testing data.
X_test = batch_zscaler(X_test, ['age', 'fnlwgt'])

############# LOAD DATA AND RUN RANDOM FOREST MODEL ##########
# I chose the Random Forest for several reasons. It's easy to use
# and generally provides strong classification performance
# with a relatively low risk of overfitting.

estimators = 500 # number of trees parameter
mss = 3 # mininum samples split parameter

# Setup random forest classifier
forest = sklearn.ensemble.RandomForestClassifier(n_estimators=estimators, min_samples_split=mss)

# Train on classifier
forest.fit(X_train,y_train)

# Generate prediction on the test data
forest_pred = forest.predict(X_test)

# I'm starting to wonder if it really is a good idea to drop the census weights.
# Get feature importance for random forest and graph them.

feature_importance(forest, X_train)

# Graph Confusion Matrix
make_confmat(y_test, forest_pred)

# Run ROC plotting function.
plot_auc(y_test, forest_pred)

# Create dataframe showing predicted values vs. actual values
pred_df = pd.DataFrame(y_test)
pred_df['prediction'] = forest_pred

# The Random forest got 85 % accuracy.
# Though the ratio of Postives to Negatives is about right, some classification errors were made.
# About 2/3ds of the classification errors were False Negatives. This contributes to a precision score
# of 73% that is 12% below the overall model accuracy.
# The starting F1 score is .67, not great but it's a start.
# I would consider just removing the census weights, but the Feature Importance appears relatively
# high for the census weights.
# AUC is .78, showing room for improvement.

########## RUN SUPPORT VECTOR MACHINES MODEL ########
# I chose the SVM because I suspect that the different
# attributes of this dataset could be interrelated in ways
# that are otherwise linearly inseperable.

svm_clf = svm.SVC()

# Train on classifier
svm_clf.fit(X_train,y_train)

# Generate prediction on the test data
svm_pred = svm_clf.predict(X_test)

# Show model results analysis
plot_auc(y_test, svm_pred)
make_confmat(y_test, svm_pred)

# SVM classifier with default settings results in 87% accuracy.
# Shows signicant imporvement in precision, contributing to better overall accuracy.
# The model has low Reliability when predicting positive class instances, as it only discovered
# slightly more than half of the cases where incomes were over 50k.
# Though the SVM classifier declared roughly the same amount of false negatives as the
# Random Forest, it was significantly better at avoiding false positives.
# The F1 score is also better at .7.
# The AUC score is unchanged, but the ROC curve does show a very slight improvement.

##### Let's reload the data and see how the models do without the 'age' column binned.

############ LOAD THE DATA ##############################
df = pd.read_csv('Berkowitz-M02-Dataset.csv')

print(df.columns)

####### RERUN MODELS WITHOUT CATEGORICAL VARIABLES ############
# Use list to specify columns to ignore. I'm removing the unnamed column since that's
# just duplicated indexes and the 50k column is the target variable.
# Also the hours-per-week bins to see how that affects performance.
exclude_cols = ['is_>50K', 'Unnamed: 0', 'is_works-34-45h', 'is_works-45-55h',
       'is_works-77-88h', 'is_works-23-34h', 'is_works-55-66h',
       'is_works-66-77h', 'is_works-0-11h', 'is_works-88-100h']

# Set y to the target variable.
y = df["is_>50K"]

# Set X to the dataset, with the excluded columns excluded.
X = df.loc[:, ~df.columns.isin(exclude_cols)]

############ SPLIT AND SCALE THE DATA ###################

# Extract features names from column titles.
features = X.columns

# Split data into training and test data using 40 percent for testing.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.40, random_state=42)

# Run the Z scaler on the training data.
X_train = batch_zscaler(X_train, ['age', 'fnlwgt'])

# Run the Z scaler on the testing data.
X_test = batch_zscaler(X_test, ['age', 'fnlwgt'])

############# LOAD DATA AND RUN RANDOM FOREST MODEL ##########
estimators = 500 # number of trees parameter
mss = 3 # mininum samples split parameter

# Setup random forest classifier
forest = sklearn.ensemble.RandomForestClassifier(n_estimators=estimators, min_samples_split=mss)

# Train on classifier
forest.fit(X_train,y_train)

# Generate prediction on the test data
forest_pred = forest.predict(X_test)

# Get feature importance for random forest and graph them.
feature_importance(forest, X_train)

# Graph Confusion Matrix
make_confmat(y_test, forest_pred)

# Run ROC plotting function.
plot_auc(y_test, forest_pred)

# Create dataframe showing predicted values vs. actual values
pred_df = pd.DataFrame(y_test)
pred_df['prediction'] = forest_pred

# The Random Forest model performs the roughly same in the dataset
# without the ages binned as it does when the ages are binned.
# The F1 score is about the same and the confusion matrix counts,
# Precision and Recall are also largely unaffected by the bins.
# AUC is .78, not showing a significant impact with the bins removed.

########## RUN SUPPORT VECTOR MACHINES MODEL #####
svm_clf = svm.SVC()

# Train on classifier
svm_clf.fit(X_train,y_train)

# Generate prediction on the test data
svm_pred = svm_clf.predict(X_test)

# Show model results analysis
plot_auc(y_test, svm_pred)
make_confmat(y_test, svm_pred)

# Though the SVM model performed relatively well with the age-binned dataset,
# the lack of bins has resulted in a much higher rate of false negatives as well
# as false positives. The F1 score of .66 is the worst of the four model/dataset
# variants. AUC has also shown a slight negative impact, shrinking to .75.
# This is reflected by a slight change in the curve.

############## SUMMARY ##############################################
# Cleaning and outlier removal was performed on the Census Dataset.
# Two classifiers were tested, the Random Forest and SVM.
# Each classifier was run on one dataset with the 'age' column binned
# and one dataset in which the the 'age' column was numeric to test
# the impact of the bins on each classifiers' performance.
# The performance of the random forest model satisfactory, showing
# 85% accuracy with both binned and non-binned datasets.
# On the other hand, the SVM surpassed both Random Forest Classifiers
# when the bins were added.

# The SVM classifier showed the strongest performance and best fit
# of all the models tested here when run on the binned dataset.
# This is in contrast to the Random Forest, which did not appear
# to be impacted by bins. This suggests that the SVM could be a very
# practical tool for datasets where significant feature engineering
# has been performed.

# CODE REFERENCES:
# Dictionary processing function adapted from:
# Title: How to get the current time in Python
# Author: TigerhawkT3
# Date: 11/26/2017
# Code version: N/A
# Availability: https://stackoverflow.com/a/40823642

# Flagging values in mulitple columns (answer to StackOverflow question I asked)
# Title: Using df.loc to flag values in multiple columns at once
# Author: coldspeed
# Date: 2/6/2018
# Code version: N/A
# Availability: https://stackoverflow.com/a/54545338

# Binning function adapted from:
# Title: L06-A-1 Normalizing and Binning in Python
# Author: Zacharias Voulgaris
# Date: NA
# Code version: N/A
# Availability: Not publicly available.

# Feature importance/Confusion matrix function adapted from:
# Title: Python Machine Learning 2nd ed.
# Author: Sebastian Rashka
# Date: 11/19/2018
# Available from: https://github.com/rasbt/python-machine-learning-book-2nd-edition/blob/master/code/ch04/ch04.ipynb

# Confusion Matrix/ROC/AUC code adapted from:
# Title: L09-AccuracyMeasures.py
# Author: Zacharias Voulgaris
# Date: NA
# Code version: N/A
# Availability: Not publicly available.


#!/usr/bin/env python3
# -*- mode: python; coding: utf-8 -*-

import logging
from datetime import datetime, timedelta
import sys
import requests
from requests.compat import urljoin
from requests_oauthlib import OAuth1Session
from threading import RLock

sys.version_info >= (3, 0) or exit('Python 3 required')

__version__ = '0.10.4'

_LOGGER = logging.getLogger(__name__)

TELLDUS_LIVE_API_URL = 'https://api.telldus.com/json/'
TELLDUS_LIVE_REQUEST_TOKEN_URL = 'https://api.telldus.com/oauth/requestToken'
TELLDUS_LIVE_AUTHORIZE_URL = 'https://api.telldus.com/oauth/authorize'
TELLDUS_LIVE_ACCESS_TOKEN_URL = 'https://api.telldus.com/oauth/accessToken'

TELLDUS_LOCAL_API_URL = 'http://{host}/api/'
TELLDUS_LOCAL_REQUEST_TOKEN_URL = 'http://{host}/api/token'
TELLDUS_LOCAL_REFRESH_TOKEN_URL = 'http://{host}/api/refreshToken'

TIMEOUT = timedelta(seconds=10)

UNNAMED_DEVICE = 'NO NAME'

# Tellstick methods
# pylint:disable=invalid-name
TURNON = 1
TURNOFF = 2
BELL = 4
TOGGLE = 8
DIM = 16
LEARN = 32
UP = 128
DOWN = 256
STOP = 512
RGBW = 1024
THERMOSTAT = 2048

SUPPORTED_METHODS = (
    TURNON |
    TURNOFF |
    DIM |
    UP |
    DOWN |
    STOP)

METHODS = {
    TURNON: 'turnOn',
    TURNOFF: 'turnOff',
    BELL: 'bell',
    TOGGLE: 'toggle',
    DIM: 'dim',
    LEARN: 'learn',
    UP: 'up',
    DOWN: 'down',
    STOP: 'stop',
    RGBW: 'rgbw',
    THERMOSTAT: 'thermostat'
}

# Sensor types
TEMPERATURE = 'temperature'
HUMIDITY = 'humidity'
RAINRATE = 'rrate'
RAINTOTAL = 'rtot'
WINDDIRECTION = 'wdir'
WINDAVERAGE = 'wavg'
WINDGUST = 'wgust'
UV = 'uv'
WATT = 'watt'
LUMINANCE = 'lum'
DEW_POINT = 'dewp'
BAROMETRIC_PRESSURE = 'barpress'

BATTERY_LOW = 255
BATTERY_UNKNOWN = 254
BATTERY_OK = 253

SUPPORTS_LOCAL_API = ['TellstickZnet', 'TellstickNetV2']

def supports_local_api(device):
    """Return true if the device supports local access."""
    return any(dev in device
               for dev in SUPPORTS_LOCAL_API)

class LocalAPISession(requests.Session):
    """Connect directly to the device."""

    def __init__(self, host, application, access_token=None):
        super().__init__()
        self.url = TELLDUS_LOCAL_API_URL.format(host=host)
        self._host = host
        self._application = application
        self.request_token = None
        self.token_timestamp = None
        self.access_token = access_token
        if access_token:
            self.headers.update(
                {'Authorization': 'Bearer {}'.format(self.access_token)})
            self.refresh_access_token()

    @property
    def authorize_url(self):
        """Retrieve URL for authorization."""
        try:
            response = self.put(
                TELLDUS_LOCAL_REQUEST_TOKEN_URL.format(host=self._host),
                data={'app': self._application},
                timeout=TIMEOUT.seconds)
            response.raise_for_status()
            result = response.json()
            self.request_token = result.get('token')
            return result.get('authUrl')
        except (OSError, ValueError) as e:
            _LOGGER.error('Failed to retrieve authorization URL: %s', e)

    def authorize(self):
        """Perform authorization."""
        try:
            response = self.get(
                TELLDUS_LOCAL_REQUEST_TOKEN_URL.format(host=self._host),
                params=dict(token=self.request_token),
                timeout=TIMEOUT.seconds)
            response.raise_for_status()
            result = response.json()
            if 'token' in result:
                self.access_token = result['token']
                self.headers.update(
                    {'Authorization': 'Bearer {}'.format(self.access_token)})
                self.token_timestamp = datetime.now()
                token_expiry = datetime.fromtimestamp(result.get('expires'))
                _LOGGER.debug('Token expires %s', token_expiry)
                return True
        except OSError as e:
            _LOGGER.error('Failed to authorize: %s', e)

    def refresh_access_token(self):
        """Refresh api token"""
        try:
            response = self.get(
                TELLDUS_LOCAL_REFRESH_TOKEN_URL.format(host=self._host))
            response.raise_for_status()
            result = response.json()
            self.access_token = result.get('token')
            self.token_timestamp = datetime.now()
            token_expiry = datetime.fromtimestamp(result.get('expires'))
            _LOGGER.debug('Token expires %s', token_expiry)
            return True
        except OSError as e:
            _LOGGER.error('Failed to refresh access token: %s', e)

    def authorized(self):
        """Return true if successfully authorized."""
        return self.access_token

    def maybe_refresh_token(self):
        """Refresh access_token if expired."""
        if self.token_timestamp:
            age = datetime.now() - self.token_timestamp
            if age > timedelta(seconds=(12 * 60 * 60)):  # 12 hours
                self.refresh_access_token()

class LiveAPISession(OAuth1Session):
    """Connection to the cloud service."""

    # pylint: disable=too-many-arguments
    def __init__(self,
                 public_key,
                 private_key,
                 token=None,
                 token_secret=None,
                 application=None):
        super().__init__(public_key, private_key, token, token_secret)
        self.url = TELLDUS_LIVE_API_URL
        self.access_token = None
        self.access_token_secret = None
        if application:
            self.headers.update({'X-Application': application})

    @property
    def authorize_url(self):
        """Retrieve URL for authorization."""
        _LOGGER.debug('Fetching request token')
        try:
            self.fetch_request_token(
                TELLDUS_LIVE_REQUEST_TOKEN_URL, timeout=TIMEOUT.seconds)
            _LOGGER.debug('Got request token')
            return self.authorization_url(TELLDUS_LIVE_AUTHORIZE_URL)
        except (OSError, ValueError) as e:
            _LOGGER.error('Failed to retrieve authorization URL: %s', e)

    def authorize(self):
        """Perform authorization."""
        try:
            _LOGGER.debug('Fetching access token')
            token = self._fetch_token(
                TELLDUS_LIVE_ACCESS_TOKEN_URL, timeout=TIMEOUT.seconds)
            _LOGGER.debug('Got access token')
            self.access_token = token['oauth_token']
            self.access_token_secret = token['oauth_token_secret']
            _LOGGER.debug('Authorized: %s', self.authorized)
            return self.authorized
        except (OSError, ValueError) as e:
            _LOGGER.error('Failed to authorize: %s', e)

    def maybe_refresh_token(self):
        """Refresh access_token if expired."""
        pass

class LocalUDPSession():
    TELLSTICK_SUCCESS = 0
    TELLSTICK_ERROR_DEVICE_NOT_FOUND = -3
    TELLSTICK_ERROR_UNKNOWN = -99

    def __init__(self, devicemanager, logger=None):
        self._request = None
        self._LOGGER = logger or logging.getLogger(__name__)
        self.devicemanager = devicemanager
        self.response = None
        self._exception = None
        self.status_code = None
        self.headers = {}
        self._json = None
        self.url = 'http://tellstick/'
        self.authorized = True

    def __str__(self):
        return self.response

    def devices(self, *args, params=None, timeout=None):
        """ creates device json """
        if args[0] == "list":
            devices = {"device": self.devicemanager.listdevices()}
            self._LOGGER.debug("Devices: %s: ", devices)
            self._json = devices
            return self.TELLSTICK_SUCCESS
        else:
            self._json = {"error": "Internal server error"}
            raise AttributeError

    def sensors(self, *args, params=None, timeout=None):
        """ creates sensor json """
        if args[0] == "list":
            sensor = {"sensor": self.devicemanager.listsensors()}
            self._LOGGER.debug("Sensor: %s: ", sensor)
            self._json = sensor
            self._LOGGER.debug("_json: %s: ", self._json)
            return self.TELLSTICK_SUCCESS
        else:
            self._json = {"error": "Internal server error"}
            raise AttributeError

    def sensor(self, *args, params=None, timeout=None):
        self._LOGGER.debug("Sensor id: %s ", params['id'])
        s = self.devicemanager.sensor(params['id'])
        self._LOGGER.debug("Got sensor: %s ", s)
        if s.isSensor():
            if args[0] == "info":
                self._json = s.deviceInfo()
                return self.TELLSTICK_SUCCESS
            else:
                self._json = {"error": "Internal server error"}
                raise AttributeError
        else:
            self._json = {"error": "Internal server error"}
            raise AttributeError

    def device(self, *args, params=None, timeout=None):
        """ runnns command on device """
        self._LOGGER.debug("Device id: %s ", params['id'])
        d = self.devicemanager.device(params['id'])
        self._LOGGER.debug("Got device: %s ", d)
        if d.isDevice():
            if args[0] == "turnOn":
                d.command(TURNON)
                self._json = {"status": "success"}
                return self.TELLSTICK_SUCCESS

            elif args[0] == "turnOff":
                d.command(TURNOFF)
                self._json = {"status": "success"}
                return self.TELLSTICK_SUCCESS
            elif args[0] == "info":
                self._json = d.deviceInfo()
                return self.TELLSTICK_SUCCESS
            else:
                self._json = {"error": "Internal server error"}
                raise AttributeError
        else:
            self._json = {"error": "Internal server error"}
            raise AttributeError

    def get(self, url, params=None, timeout=None):
        """ request get faker """
        self._LOGGER.debug("url: %s: ", url)
        self._request = url[len(self.url):].split('/')
        self._LOGGER.debug("get() _request[0]: %s: ", self._request[0])
        self._LOGGER.debug("get() _request[1:]: %s: ", self._request[1:])
        self._LOGGER.debug("get() params: %s: ", params)
        self.headers['content-type'] = "application/json; charset=utf-8"
        try:
            self.response = getattr(self,
                                    "%s" % self._request[0]
                                    )(*self._request[1:],
                                      params=params,
                                      timeout=timeout)
        except AttributeError:
            self._exception = "500 Internal server error %s" % self._request
            self.status_code = 500
            self._json = {"error": "Internal server error"}
            return self
        self.status_code = 200
        self._LOGGER.debug("get() _json: %s: ", self._json)
        return self

    @staticmethod
    def raise_for_status():
        """ pass status exception """
        pass

    def json(self):
        """ returns json """
        return self._json

    @staticmethod
    def maybe_refresh_token():
        """Refresh access_token if expired."""
        pass

class DefaultCallbackDispatcher(object):
    def __init__(self):
        super(DefaultCallbackDispatcher, self).__init__()

    def on_callback(self, callback, *args):
        callback(*args)

class AsyncioCallbackDispatcher(object):
    """Dispatcher for use with the event loop available in Python 3.4+.
    Callbacks will be dispatched on the thread running the event loop. The loop
    argument should be a BaseEventLoop instance, e.g. the one returned from
    asyncio.get_event_loop().
    """
    def __init__(self, loop):
        super(AsyncioCallbackDispatcher, self).__init__()
        self._loop = loop
        _LOGGER.debug("AsyncioCallbackDispatcher enabled with loop: %s", loop)

    def on_callback(self, callback, *args):
        _LOGGER.debug("AsyncioCallbackDispatcher called")
        self._loop.call_soon_threadsafe(callback, *args)

class Session:
    """Tellduslive session."""

    # pylint: disable=too-many-arguments
    def __init__(self,
                 public_key=None,
                 private_key=None,
                 token=None,
                 token_secret=None,
                 host=None,
                 application=None,
                 listen=False,  # listen for local UDP broadcasts
                 callback=None,  # callback for asynchrounous sensor updates
                 config=None,  # config for localUDPSession and async_listner
                 callback_dispatcher=None):

        if callback_dispatcher:
            self._callback_dispatcher = callback_dispatcher
        else:
            self._callback_dispatcher = DefaultCallbackDispatcher()

        _LOGGER.info('%s version %s', __name__, __version__)
        if not(all([public_key,
                    private_key,
                    token,
                    token_secret]) or
               all([listen]) or
               all([host, token]) or
               all([host, listen])):
            raise ValueError('Missing configuration')

        self._state = {}
        self._lock = RLock()
        if listen:
            from tellsticknet import devicemanager
            self._devicemanager = devicemanager.Tellstick(host=host,
                                                          logger=_LOGGER,
                                                          config=listen)

        host = host or (listen
                        if isinstance(listen, str)
                        else None)

        self._session = (
            LocalAPISession(host,
                            application,
                            token) if host and token and not public_key else
            LiveAPISession(public_key,
                           private_key,
                           token,
                           token_secret,
                           application) if (public_key and
                                            private_key and
                                            token and
                                            token_secret) else
            LocalUDPSession(self._devicemanager))

        if listen:
            _LOGGER.debug("Callback functions is: %s", callback)
            self.update()
            for d in self.devices:
                if not d.is_sensor:
                    self._devicemanager.adddevice({'name': d.name,
                                                   'id': d.device_id,
                                                   'parameters': d.parameters,
                                                   'protocol': d.protocol,
                                                   'model': d.model,
                                                   'client_id': d.client_id})
            self._setup_async_listener(self._devicemanager, callback)

    def _setup_async_listener(self, devicemanager, callback):
        """Starts listening for asynchronous UDP packets on the
        local network. If host is None, autodiscovery will be used."""
        def got(device):
            """Callback when ascynhronous packet is received.
            N.B. will be called in another thread."""
            local_id = ()
            with self._lock:
                callbackdevice = None
                """ check i device is a sensor """
                if 'sensorId' in device:
                    local_id = (device['protocol'],
                                device['model'],
                                str(device['sensorId']))
                    _LOGGER.debug('Received asynchronous packet %s:%s:%s',
                                  *local_id)
                    _LOGGER.debug('Received asynchronous data %s from %s',
                                  device['data'], local_id)

                    sensor = next((sensor
                                   for sensor in self.sensors
                                   if ((sensor.protocol,
                                        sensor.model,
                                        str(sensor.sensorId)) == local_id)),
                                  None)

                    if not sensor:
                        _LOGGER.info('Found no corresponding device on server'
                                     'for packet %s:%s:%s %s', *local_id,
                                     'new sensor added')
                        self._state.update({'_' + str(device['id']): device})

                        sensor = next((sensor
                                       for sensor in self.sensors
                                       if ((sensor.protocol,
                                            sensor.model,
                                            str(sensor.sensorId)
                                            ) == local_id)),
                                      None)

                    _LOGGER.debug('Got asynchronous update from sensor %s',
                                  sensor.name)

                    sensor.device.update({"data": device['data']})
                    callbackdevice = sensor.device
                else:
                    for param in device.get('parameters'):
                        if param.get('name') == 'unit':
                            unit = param.get('value')
                        elif param.get('name') == 'house':
                            house = param.get('value')
                    local_id = (house, unit)
                    _LOGGER.debug('Received asynchronous data %s from %s',
                                  device, local_id)
                    for dev in self.devices:
                        if dev.parameters:
                            for param in dev.parameters:
                                if param.get('name') == 'unit':
                                    unit = param.get('value')
                                elif param.get('name') == 'house':
                                    house = param.get('value')
                            device_id = (house, unit)
                            if device_id == local_id:
                                _LOGGER.debug('Found device %s from %s',
                                              dev.device, device_id)
                                break
                    if not dev:
                        _LOGGER.info('Found no corresponding device on server'
                                     'for packet %s %s', local_id,
                                     'new device  added')
                        self._state.update({str(device['id']): device})

                        dev = next((dev
                                    for dev in self.devices
                                    if (dev.device_id == device['id'])), None)

                    _LOGGER.debug('Got asynchronous update from device %s',
                                  dev.name)
                    dev.device.update({'state': device.get('state')})
                    callbackdevice = dev.device
                _LOGGER.debug("callback device id %s",
                              callbackdevice.get('id'))
                self._callback_dispatcher.on_callback(callback,
                                                      callbackdevice)

        _LOGGER.info('Starting asynchronous listener thread')
        devicemanager.async_listen(callback=got)

    @property
    def authorize_url(self):
        """Retrieve URL for authorization."""
        return self._session.authorize_url

    def authorize(self):
        """Perform authorization."""
        return self._session.authorize()

    @property
    def access_token(self):
        """Return access token."""
        return self._session.access_token

    @property
    def is_authorized(self):
        """Return true if successfully authorized."""
        return self._session.authorized

    @property
    def access_token_secret(self):
        """Return the token secret."""
        return self._session.access_token_secret

    def _device(self, device_id):
        """Return the raw representaion of a device."""
        with self._lock:
            return self._state.get(device_id)

    def _request(self, path, **params):
        """Send a request to the Tellstick Live API."""
        try:
            self._session.maybe_refresh_token()
            url = urljoin(self._session.url, path)
            _LOGGER.debug('Request %s %s', url, params)
            response = self._session.get(url,
                                         params=params,
                                         timeout=TIMEOUT.seconds)
            response.raise_for_status()
            _LOGGER.debug('Response %s %s %s',
                          response.status_code,
                          response.headers['content-type'],
                          response.json())
            response = response.json()
            if 'error' in response:
                raise OSError(response['error'])
            return response
        except OSError as error:
            _LOGGER.warning('Failed request: %s', error)

    def execute(self, method, **params):
        """Make request, check result if successful."""
        with self._lock:
            response = self._request(method, **params)
            return response and response.get('status') == 'success'

    def _request_devices(self):
        """Request list of devices from server."""
        res = self._request('devices/list',
                            supportedMethods=SUPPORTED_METHODS,
                            includeIgnored=0)
        return res.get('device') if res else None

    def _request_device(self, id):
        """Request list of devices from server."""
        res = self._request('device/info',
                            id=id)
        return res if res else None

    def _request_sensor(self, id):
        """Request list of devices from server."""
        res = self._request('sensor/info',
                            id=id)
        return res if res else None

    def _request_sensors(self):
        """Request list of sensors from server."""
        res = self._request('sensors/list',
                            includeValues=1,
                            includeScale=1,
                            includeIgnored=0)
        return res.get('sensor') if res else None

    def update(self):
        """Updates all devices and sensors from server."""
        with self._lock:
            def collect(devices, is_sensor=False):
                """Update local state.
                N.B. We prefix sensors with '_',
                since apparently sensors and devices
                do not share name space and there can
                be collissions.
                FIXME: Remove this hack."""
                self._state.update({'_' * is_sensor + str(device['id']): device
                                    for device in devices or {}
                                    if device['name'] and
                                    not (is_sensor and
                                   'data' not in device)})

            devices = self._request_devices()
            for i, d in enumerate(devices):
                if d.get('id') in list(self.device_ids):
                    _LOGGER.debug("already known device")
                    req_dev = self.device(d.get('id'))
                    d.update({'parameters': req_dev.parameters})
                    d.update({'protocol': req_dev.protocol})
                    d.update({'model': req_dev.model})
                    d.update({'client_id': req_dev.client_id})
                    devices[i].update(d)
                else:
                    _LOGGER.debug("Getting protocol and parameters",
                                  "for new device")
                    req_dev = self._request_device(d.get('id'))
                    d.update({'parameters': req_dev.get('parameter')})
                    d.update({'protocol': req_dev.get('protocol')})
                    d.update({'model': req_dev.get('model')})
                    d.update({'client_id': req_dev.get('client')})
                    devices[i].update(d)
            collect(devices)

            sensors = self._request_sensors()
            collect(sensors, True)

            return (devices is not None and
                    sensors is not None)

    def device(self, device_id):
        """Return a device object."""
        return Device(self, device_id)

    @property
    def sensors(self):
        """Return only sensors.
        FIXME: terminology device vs device."""
        return (device
                for device in self.devices
                if device.is_sensor)

    @property
    def devices(self):
        """Request representations of all devices."""
        return (self.device(device_id) for device_id in self.device_ids)

    @property
    def device_ids(self):
        """List of known device ids."""
        with self._lock:
            return self._state.keys()

class Device:
    """Tellduslive device."""

    def __init__(self, session, device_id):
        self._session = session
        self._device_id = device_id

    def __str__(self):
        if self.is_sensor:
            items = ', '.join(str(item) for item in self.items)
            return 'Sensor #{id:>9} {name:<20} ({items})'.format(
                id=self.device_id,
                name=self.name or UNNAMED_DEVICE,
                items=items)
        else:
            return ('Device #{id:>9} {name:<20} '
                    '({state}:{value}) [{methods}]').format(
                        id=self.device_id,
                        name=self.name or UNNAMED_DEVICE,
                        state=self._str_methods(self.state),
                        value=self.statevalue,
                        methods=self._str_methods(self.methods))

    def __getattr__(self, name):
        if (self.device and
            name in ['name', 'state', 'battery', 'unit', 'house',
                     'model', 'protocol', 'parameters', 'client_id',
                     'lastUpdated', 'methods', 'data', 'sensorId']):
            return self.device.get(name)

    @property
    def device(self):
        """Return the raw representation of the device."""
        # pylint: disable=protected-access
        return self._session._device(self.device_id)

    @property
    def device_id(self):
        """Id of device."""
        return self._device_id

    @staticmethod
    def _str_methods(val):
        """String representation of methods or state."""
        res = []
        for method in METHODS:
            if val & method:
                res.append(METHODS[method].upper())
        return "|".join(res)

    def _execute(self, command, **params):
        """Send command to server and update local state."""
        params.update(id=self.device_id)
        # Corresponding API methods
        method = 'device/{}'.format(METHODS[command])
        if self._session.execute(method, **params):
            self.device['state'] = command
            return True

    @property
    def is_sensor(self):
        """Return true if this is a sensor."""
        return 'data' in self.device

    @property
    def statevalue(self):
        """State value of device."""
        val = self.device.get('statevalue')
        return val if val and val != 'unde' else 0

    @property
    def is_on(self):
        """Return true if device is on."""
        return (self.state == TURNON or
                self.state == DIM)

    @property
    def is_down(self):
        """Return true if device is down."""
        return self.state == DOWN

    @property
    def dim_level(self):
        """Return current dim level."""
        try:
            return int(self.statevalue)
        except (TypeError, ValueError):
            return None

    def turn_on(self):
        """Turn device on."""
        return self._execute(TURNON)

    def turn_off(self):
        """Turn device off."""
        return self._execute(TURNOFF)

    def dim(self, level):
        """Dim device."""
        if self._execute(DIM, level=level):
            self.device['statevalue'] = level
            return True

    def up(self):
        """Pull device up."""
        return self._execute(UP)

    def down(self):
        """Pull device down."""
        return self._execute(DOWN)

    def stop(self):
        """Stop device."""
        return self._execute(STOP)

    @property
    def items(self):
        """Return sensor items for sensor."""
        return (SensorItem(item) for item in self.data) if self.data else []

    def item(self, name, scale):
        """Return sensor item."""
        return next((item for item in self.items
                     if (item.name == name and
                         int(item.scale) == int(scale))), None)

    def value(self, name, scale):
        """Return value of sensor item."""
        return self.item(name, scale).value

class SensorItem:
    # pylint: disable=too-few-public-methods, no-member
    """Reference to a sensor data item."""
    def __init__(self, data):
        vars(self).update(data)

    def __str__(self):
        return '{name}={value}'.format(
            name=self.name, value=self.value)

def read_credentials():
    from sys import argv
    from os.path import join, dirname, expanduser
    for directory in [
            dirname(argv[0]),
            expanduser('~')]:
        try:
            with open(join(directory, '.tellduslive.conf')) as config:
                return dict(
                    x.split(': ')
                    for x in config.read().strip().splitlines()
                    if not x.startswith('#')
                    if not x.startswith('local'))
        except OSError:
            continue
    return {}

if __name__ == '__main__':
    """Dump configured devices and sensors."""
    logging.basicConfig(level=logging.INFO)
    credentials = read_credentials()
    session = Session(**credentials)
    session.update()
    print('Devices\n'
          '-------')
    for device in session.devices:
        print(device)
        for item in device.items:
            print('- {}'.format(item))


import pandas as pd
import json
from collections import Counter

def json_readup(path):
    with open(path) as data_file:
        data = json.load(data_file)
    return data

def change_the_dict(dict_in):

    for key in dict_in.keys():
        dict_in[key] = set(dict_in[key])
    return dict_in

def dataset_integrator(input_data, artist_dict, flow_dict, chart_dict, track_dict, album_dict):

    user_ids = map(lambda s: str(s), input_data["user_id"].tolist())
    artists = input_data["artist_id"].tolist()
    tracks = input_data["media_id"].tolist()
    albums = input_data["album_id"].tolist()

    artists_unique = artist_dict.keys()
    tracks_unique = track_dict.keys()
    albums_unique = album_dict.keys()
    flows_unique = flow_dict.keys()
    charts_unique = chart_dict.keys()

    new_artist, artist_aggregates = [],[]
    new_flow, flow_aggregates = [],[]
    new_track, track_aggregates = [],[]
    new_chart, chart_aggregates = [], []
    new_album, album_aggregates = [],[]

    index = 0

    for i in range(0, len(user_ids)):
        index = index + 1
        print(index)
        if index % 20000 == 0:

            output = pd.DataFrame()
            output["artist_liked"] = new_artist
            output["artist_like_count"] = artist_aggregates

            output["in_flow"] = new_flow
            output["number_of_items_in_flow"] = flow_aggregates

            output["track_liked"] = new_track
            output["number_of_tracks_liked"] = track_aggregates

            output["chart_liked"] = new_chart
            output["number_of_charts_liked"] = chart_aggregates

            output["album_liked"] = new_album
            output["number_of_albums_liked"] = album_aggregates

            output.to_csv("./raw_dataset/train_pieces/train_" + str(index) + ".csv",sep = ",", index = None)

            new_artist, artist_aggregates = [],[]
            new_flow, flow_aggregates = [],[]
            new_track, track_aggregates = [],[]
            new_chart, chart_aggregates = [], []
            new_album, album_aggregates = [],[]

        if user_ids[i] in artists_unique:
            if artists[i] in artist_dict[user_ids[i]]:
                new_artist = new_artist + [1]

            else:
                new_artist = new_artist + [0]
            artist_aggregates = artist_aggregates  + [len(artist_dict[user_ids[i]])]
        else:
            new_artist = new_artist + [-1]
            artist_aggregates = artist_aggregates  + [0]

        if user_ids[i] in flows_unique:
            if tracks[i] in flow_dict[user_ids[i]]:
                new_flow = new_flow + [1]
            else:
                new_flow = new_flow+ [0]
            flow_aggregates = flow_aggregates  + [len(flow_dict[user_ids[i]])]
        else:
            new_flow = new_flow + [-1]
            flow_aggregates = flow_aggregates  + [0]

        if user_ids[i] in tracks_unique:
            if tracks[i] in track_dict[user_ids[i]]:
                new_track = new_track + [1]
            else:
                new_track = new_track + [0]
            track_aggregates = track_aggregates  + [len(track_dict[user_ids[i]])]
        else:
            new_track = new_track + [-1]
            track_aggregates = track_aggregates  + [0]

        if user_ids[i] in charts_unique:
            if tracks[i] in chart_dict[user_ids[i]]:
                new_chart = new_chart + [1]
            else:
                new_chart = new_chart + [0]
            chart_aggregates = chart_aggregates  + [len(chart_dict[user_ids[i]])]
        else:
            new_chart = new_chart + [-1]
            chart_aggregates = chart_aggregates  + [0]

        if user_ids[i] in albums_unique:
            if albums[i] in album_dict[user_ids[i]]:
                new_album = new_album + [1]
            else:
                new_album = new_album + [0]
            album_aggregates = album_aggregates  + [len(album_dict[user_ids[i]])]
        else:
            new_album = new_album + [-1]
            album_aggregates = album_aggregates  + [0]

    output = pd.DataFrame()

    output["artist_liked"] = new_artist
    output["artist_like_count"] = artist_aggregates

    output["in_flow"] = new_flow
    output["number_of_items_in_flow"] = flow_aggregates

    output["track_liked"] = new_track
    output["number_of_tracks_liked"] = track_aggregates

    output["chart_liked"] = new_chart
    output["number_of_charts_liked"] = chart_aggregates

    output["album_liked"] = new_album
    output["number_of_albums_liked"] = album_aggregates
    return output, index

def run_this():

    test = pd.read_csv("./raw_dataset/test.csv",index_col = None)
    train = pd.read_csv("./raw_dataset/train_deduplicated.csv",index_col = None)

    artist_dict = json_readup("./large_dumps/user_artists.json")
    flow_dict = json_readup("./large_dumps/user_flow.json")
    chart_dict = json_readup("./large_dumps/user_charts.json")
    track_dict = json_readup("./large_dumps/user_tracks.json")
    album_dict = json_readup("./large_dumps/user_albums.json")

    artist_dict = change_the_dict(artist_dict)
    flow_dict = change_the_dict(flow_dict)
    chart_dict = change_the_dict(chart_dict)
    track_dict = change_the_dict(track_dict)
    album_dict = change_the_dict(album_dict)

    new_test, test_index = dataset_integrator(test, artist_dict, flow_dict, chart_dict, track_dict, album_dict)
    new_train, train_index = dataset_integrator(train, artist_dict, flow_dict, chart_dict, track_dict, album_dict)

    new_test.to_csv("./raw_dataset/test_likes.csv",sep = ",", index = None)
    new_train.to_csv("./raw_dataset/train_pieces/train_" + str(train_index) + ".csv",sep = ",", index = None)

run_this()


import challenge3
from binascii import unhexlify

def decodeLines(fileName):
    with open(fileName, 'r') as f:
        for line in f:
            if line[-1] == '\n':
                line = line[:-1]
            yield unhexlify(line)

#def detectSingleCharXor(fileName):
#
#    scores, msg = [], ""
#
#    with open(fileName, 'r') as f:
#
#        for line in f:
#
#            if line[-1] == '\n':
#                line = line[:-1]
#            decoded = breakSingleByteXor(unhexlify(line))
#            # decoded[0] is the message, decoded[2] is the score
#            scores.append(decoded[2])
#            if decoded[2] >= max(scores):
#                msg = decoded[0]
#
#    return msg

def detectSingleByteXor(lines):

    maxScore = -1
    for line in lines:
        decoded = challenge3.breakSingleByteXor(line)
        if decoded[2] >= maxScore:
            maxScore = decoded[2]
            msg = decoded[0]
    return msg

if __name__=='__main__':

    #msg = detectSingleCharXor("4.txt")
    #print(msg)

    lines = decodeLines('4.txt')
    msg = detectSingleByteXor(lines)
    print(msg)


import argparse
import logging
import random
import time
import math
import os
import numpy as np
import PIL
from tqdm import tqdm
import tensorflow as tf
from PIL import Image, ImageDraw, ImageOps
CUDA_VISIBLE_DEVICES = []

dest_path = None
prefix = None

#SETTINGS: These values are set in set_dest_path
#number of samples per tfrecord, one sample equals one sequence
samples_per_record = 20
#number of frames used before starting prediction
K = 9
#number of frames to predict
T = 10
#values copied over from original preprocessing script
prescale=337
crop_size=96
image_size=96
occup_steps=1
#number of frames per sequence, needs to be at least K+T+1
seq_length=20
#number of frames to skip between sequences
step_size=5
#if one tfrecord sequence holds multiple of (K+T+1) can loop over additional frames
#equal to having multiple sequences with only (K+T+1) frames
seq_steps=seq_length//(K+T)
#Threshold for mean of pixels over channels (between 0 and 255)
thresh = 96
#frame composing not supported for now, need to change code in calcImageTranslation
#frame_composing=2

occupancy_buffer = []
transformation_buffer = []
occlusion_buffer = []
return_clips = []
return_transformation = []
rgb_buffer = []
depth_buffer = []
segmentation_buffer = []
return_camera_rgb = []
return_camera_segmentation = []
return_camera_depth = []
idnr = 0
data_size = []
direction_buffer = []
return_direction = []
episode = 0

def set_dest_path(path, _samples_per_record, _K, _T, _image_size, _seq_length, _step_size):
    global dest_path
    global samples_per_record
    global K, T, prescale, crop_size, image_size, seq_length
    global step_size, thresh
    dest_path = path
    if not os.path.exists(path):
        os.makedirs(path)

    samples_per_record = _samples_per_record
    K = _K
    T = _T
    crop_size = _image_size
    image_size = _image_size
    seq_length = _seq_length
    step_size = _step_size

def update_episode_reset_globals(pfx):
    global prefix, occupancy_buffer, occlusion_buffer, transformation_buffer
    global return_clips, return_transformation, rgb_buffer, depth_buffer
    global segmentation_buffer, return_camera_rgb, return_camera_segmentation, return_camera_depth
    global idnr, data_size, direction_buffer, return_direction
    prefix = pfx
    occupancy_buffer = []
    transformation_buffer = []
    occlusion_buffer = []
    return_clips = []
    return_transformation = []
    rgb_buffer = []
    depth_buffer = []
    segmentation_buffer = []
    return_camera_rgb = []
    return_camera_segmentation = []
    return_camera_depth = []
    idnr = 0
    data_size = []
    direction_buffer = []
    return_direction = []

def main(img, rgb, depth, segmentation, yaw_rate, speed):
    global prefix, occupancy_buffer, occlusion_buffer, transformation_buffer
    global return_clips, return_transformation, rgb_buffer, depth_buffer
    global segmentation_buffer, return_camera_rgb, return_camera_segmentation, return_camera_depth
    global idnr, data_size, direction_buffer, return_direction

    #find the direction (left,right,straight)
    if yaw_rate < -0.5: #going left
        direction_buffer.append([1,1])
    elif yaw_rate > 0.5: #going right
        direction_buffer.append([0,1])
    else: #going straight
        direction_buffer.append([0,0])

    #resize input images
    new_size = tuple(t//8 for t in rgb.shape[:-1]) # from 1920 x 640 to 240 x 80
    data_size = [new_size[1], new_size[0]]
    new_size = tuple(data_size)
    rgb_buffer.append(transform_input(rgb, new_size, False))
    depth_buffer.append(transform_input(depth, new_size, True))
    segmentation_buffer.append(transform_input(segmentation, new_size, False))

    #generate gridmap from bounding box image, condensed version of original preprocessing script
    occupancy_img = cropAndResizeImage(img)
    occupancy_array = np.array(occupancy_img)
    if len(occupancy_array.shape) == 3:
        occupancy_array = np.mean(occupancy_array, axis=2)
    occlusion_array = createOcclusionMap(occupancy_array)
    occupancy_mask = createOccupancyMask(occupancy_img, occlusion_array, thresh)
    transformation_matrix = calcImageTranslation(occupancy_array, yaw_rate, speed)
    occupancy_buffer.append(occupancy_mask)
    occlusion_buffer.append(occlusion_array)
    transformation_buffer.append(transformation_matrix)

    #if enough frames have been buffered combine them into one sequence
    if len(occupancy_buffer) >= seq_length:
        compressMoveMapDataset(occupancy_buffer, occlusion_buffer, transformation_buffer)
        return_camera_rgb.append(np.array(rgb_buffer))
        return_camera_segmentation.append(np.array(segmentation_buffer))
        return_camera_depth.append(np.array(depth_buffer))
        return_direction.append(direction_buffer)

        #if enough sequences have been gathered combine them into one tfrecord
        if len(return_clips) >= samples_per_record:
            convert_tfrecord(np.array(return_camera_rgb),
                             np.array(return_camera_segmentation),
                             np.array(return_camera_depth), np.array(return_direction).astype(np.uint8))
            idnr += 1
            return_clips = []
            return_transformation = []
            return_camera_rgb = []
            return_camera_segmentation = []
            return_camera_depth = []
            return_direction = []

        del occupancy_buffer[:step_size]
        del occlusion_buffer[:step_size]
        del transformation_buffer[:step_size]
        del rgb_buffer[:step_size]
        del depth_buffer[:step_size]
        del segmentation_buffer[:step_size]
        del direction_buffer[:step_size]

#resize and convert images
def transform_input(responses, size, convert=True):
    responses = Image.fromarray(np.uint8(responses))
    if convert:
        return (np.array(responses.resize(size, Image.ANTIALIAS).convert('L')) / 127 - 1).astype(np.float32)
    else:
        return (np.array(responses.resize(size, Image.ANTIALIAS)) / 127 - 1).astype(np.float32)

#all following functions taken from preprocessing script, for more information see there
def cropAndResizeImage(img):
    if prescale > 0:
        img.thumbnail((prescale, prescale), PIL.Image.ANTIALIAS)
    crop_img = ImageOps.fit(img, [crop_size, crop_size])
    scaled_img = crop_img.resize((image_size, image_size), PIL.Image.ANTIALIAS)
    return scaled_img

two_pi_f = 2 * math.pi
angular_res_rad_f = two_pi_f/900.0
radial_res_meter_f = 0.2
radial_limit_meter_f = 73
grid_cell_size = 0.4
grid_cell_size_inv_f = 1 / grid_cell_size
occ_thresh_f = 96

def createOcclusionMap(gridmap, max_occluded_steps=1):
    num_cells_per_edge_ui = gridmap.shape[0]
    num_cells_per_edge_half_f = gridmap.shape[0] // 2 - 1

    occlusion_map = np.ones(gridmap.shape, dtype=np.float32)    # 0 - occluded, 1 - non occluded/visible
    start_time = time.time()

    # Angle array captures 0 to 360 degree in radians to simulate the lidar beams
    angle_array = np.arange(0,two_pi_f,angular_res_rad_f)
    # Radial array captures 0 to max distance of detection to iterate over the distance to the ego vehicle
    radial_array = np.arange(0, radial_limit_meter_f, radial_res_meter_f)
    # For performance: repeat both arrays up to the shape of the other one to do faster matrix operations
    angle_array = np.stack([angle_array]*radial_array.shape[0], axis=1)
    radial_array = np.stack([radial_array]*angle_array.shape[0], axis=0)

    # x,y grid contains all x,y-Coordinates which correlate to the given angle and radius
    xy_grid = np.empty((angle_array.shape[0], radial_array.shape[1], 2), dtype=int)
    xy_grid[:,:,0] = grid_cell_size_inv_f * np.multiply(np.cos(angle_array), radial_array) + num_cells_per_edge_half_f # 0 - x
    xy_grid[:,:,1] = grid_cell_size_inv_f * np.multiply(np.sin(angle_array), radial_array) + num_cells_per_edge_half_f # 1 - y
    xy_grid = np.clip(xy_grid, 0, int(num_cells_per_edge_ui-1))

    occluded_steps = np.zeros((xy_grid.shape[0]), dtype=np.int32)
    is_occluded_array = np.zeros((xy_grid.shape[0]), dtype=np.bool)
    occlusion_wo_occup = np.ones((xy_grid.shape[0]), dtype=np.bool)
    position_array = np.zeros((xy_grid.shape[0], 2), dtype=int)

    for radial_index in range(xy_grid.shape[1]):
        x_i = xy_grid[:, radial_index, 0]
        y_i = xy_grid[:, radial_index, 1]

        occ_f = gridmap[y_i, x_i]
        is_occupied = (occ_f < occ_thresh_f)
        is_changed = is_occupied * (1 - is_occluded_array)
        position_array[:,0] = position_array[:,0] * (1 - is_changed) + x_i * (is_changed)
        position_array[:,1] = position_array[:,1] * (1 - is_changed) + y_i * (is_changed)
        is_occluded_array = is_occluded_array + is_occupied
        is_first_pixel = (np.absolute(position_array[:,0] - x_i) <= max_occluded_steps) * (np.absolute(position_array[:,1] - y_i) <= max_occluded_steps) * is_occupied

        occlusion_map[y_i, x_i] = occlusion_map[y_i, x_i] * (1 - (is_occluded_array * (1 - is_first_pixel)))

    return occlusion_map

def createOccupancyMask(occupancy_img, occlusion_array, thresh):
    img = np.array(occupancy_img).astype(np.float32)
    if len(img.shape) == 3:
        img = np.mean(img, axis=2)
    img = (img < thresh) * 1.0
    img = draw_ego_vehicle(img)
    occupancy_mask = np.multiply(img, occlusion_array)
    return occupancy_mask

def draw_ego_vehicle(img):
    image_size = img.shape[0]
    VEHICLE_WIDTH = image_size * 4 // 96
    VEHICLE_HEIGHT = image_size * 7 // 96
    x_start = image_size // 2 - VEHICLE_WIDTH // 2
    x_end = x_start + VEHICLE_WIDTH
    y_start = image_size // 2
    y_end = y_start + VEHICLE_HEIGHT
    img[y_start:y_end, x_start:x_end] = 1
    return img

def createOcclusionImages(img, occlusion_map):
    if len(img.shape) == 2:
        img = np.reshape(img, [img.shape[0], img.shape[1], 1])
    if img.shape[2] == 1:
        img = np.concatenate([img]*3, axis=2)
        img = img + (255 - img) * 0.5
    occlusion_map = np.stack([occlusion_map] * 3, axis=2)
    occluded_img = np.multiply(img, occlusion_map)
    return occluded_img

def calcImageTranslation(img, yaw_rate, vel, frame_rate=24, combined_frames=1, gridmap_size = 45.6):
    img = np.stack([img] * 3, axis=2)
    assert(img.shape[2] == 3)
    transform_matrix = np.zeros([3, 2, 3])
    for i in range(transform_matrix.shape[0]):
        if combined_frames == 1 or (len(frames) - frame_index <= combined_frames):
            theta, dx, dy = get_transformation_parameter(img.shape[0] // (2 ** i), gridmap_size, frame_rate, vel, yaw_rate)
        else: #this part needed for frame_composing > 1, currently not supported, needs list of imgs & odometry
            vels = []
            yaw_rates = []
            for comb_index in range(combined_frames):
                vels.append(odometry_dict[get_frame_number(frames[frame_index + comb_index])]['vel'])
                yaw_rates.append(odometry_dict[get_frame_number(frames[frame_index + comb_index])]['yaw_rate'])
            theta, dx, dy = get_combined_transformation_parameter(img.shape[0] // (2 ** i), gridmap_size, frame_rate, vels,yaw_rates)

        transform_matrix[i, :] = get_STM_matrix(img.shape[0] // (2 ** i), theta, dx, dy)
    return transform_matrix

def get_transformation_parameter(imsize, gridmap_size, frame_rate, vel, yaw_rate):
    period_duration = 1.0 / frame_rate
    yaw_diff = math.radians(yaw_rate * period_duration)
    pixel_size = gridmap_size * 1.0 / imsize    # [m]
    pixel_diff = vel * period_duration * 1.0 / pixel_size
    pixel_diff_y = math.cos(yaw_diff) * pixel_diff
    pixel_diff_x = math.sin(yaw_diff) * pixel_diff
    return yaw_diff, pixel_diff_x, pixel_diff_y

def get_STM_matrix(imsize, theta, dx, dy):
    theta = -theta
    a11 = math.cos(theta)
    a12 = -math.sin(theta)
    a13 = dx / ((imsize - 1) / 2.0)
    a21 = math.sin(theta)
    a22 = math.cos(theta)
    a23 = dy / ((imsize - 1) / 2.0)
    M = np.array([[a11, a12, a13], [a21, a22, a23]])
    return M

def create_default_element(image_size, seq_length, channel_size):
    element = np.zeros([image_size, image_size, seq_length * channel_size], dtype=np.float32)
    return element

def read_frame(occup_map, occlusion_map):
    #dummy values as long as no underlying road can be extracted from carla - backward comp. with NN model
    lines_map = np.ones(shape=(occup_map.shape)) * 255
    road_map = np.ones(shape=(occup_map.shape)) * 255
    if len(occlusion_map.shape) == 3:
        occlusion_map = np.mean(occlusion_map, axis=2)
    return np.stack([occup_map*255, occlusion_map*255, lines_map, road_map, occlusion_map*255], axis=2)

def read_matrix(matrix):
    tf_matrix = np.zeros([3,8], dtype=np.float32)
    tf_matrix[:,0:3] = matrix[:,0,:]
    tf_matrix[:,3:6] = matrix[:,1,:]
    return tf_matrix

def normalize_frames(frames):
    new_frames = frames.astype(np.float32)
    new_frames //= (255 // 2)
    new_frames -= 1
    return new_frames

def get_occupancy_diff(clip):
    occup_clip = np.multiply((clip[:,:,::5] + 1)/2.0, (clip[:,:,4::5] + 1)/2.0)
    occup_diff = occup_clip[:,:,:-1] - occup_clip[:,:,1:]
    occup_diff = np.absolute(occup_diff)
    return np.sum(occup_diff) * 1.0 / occup_diff.shape[2]

def compressMoveMapDataset(occupancy_buffer, occlusion_buffer, transformation_buffer,
                           transformation_only=False, split_number = 0, split_amount = 1):
    global return_clips
    global return_transformation
    max_occup_diff = 0
    min_occup_diff = 100000
    mean_occup_diff = 0.0
    all_clips = [create_default_element(image_size, seq_length, 5) for i in range(seq_length)]
    all_transformation = [np.zeros([seq_length, 3, 8], dtype=np.float32) for i in range(seq_length)]

    step_offset = int(round(1.0 * step_size / split_amount * split_number))

    for file_index in range(len(occupancy_buffer)):
        frame = read_frame(occupancy_buffer[file_index], occlusion_buffer[file_index])
        if transformation_only:
            frame = np.zeros([1,1,1])
        transform_matrix = read_matrix(transformation_buffer[file_index])

        channel_size = frame.shape[2]

        norm_frame = normalize_frames(frame)
        for clip_index in range(len(all_clips)):
            if not transformation_only:
                all_clips[clip_index][:, :, clip_index * channel_size: (clip_index + 1) * channel_size] = norm_frame
            all_transformation[clip_index][clip_index,:,:] = transform_matrix
        if file_index >= seq_length - 1:
            if not transformation_only:
                occup_diff = get_occupancy_diff(all_clips[-1])
                max_occup_diff = max(max_occup_diff, occup_diff)
                min_occup_diff = min(min_occup_diff, occup_diff)
                mean_occup_diff = mean_occup_diff + occup_diff
            if transformation_only or occup_diff >= 0: #occup_diff >= 6 in orig. script
                if not transformation_only:
                    return_clips.append(all_clips[-1])
                return_transformation.append(all_transformation[-1])
            else:
                print("Sequence not saved due to occupancy difference of " + str(occup_diff))

        del all_clips[-1]
        del all_transformation[-1]
        all_clips.insert(0, create_default_element(image_size, seq_length, channel_size))
        all_transformation.insert(0, np.zeros([seq_length, 3, 8], dtype=np.float32))

def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def load_gridmap_onmove_tfrecord(ind, size, frame_count, useCombinedMask=False):
    seq = return_clips[ind]
    seq = np.stack(np.split(seq[:,:,:frame_count*5], frame_count, axis=2), axis=2)
    if image_size < seq.shape[0]:
        orig_size = seq.shape[0]
        start_index = (orig_size - size) // 2
        end_index = start_index + size
        seq = seq[start_index:end_index,start_index:end_index]

    input_seq = seq[:,:,:-1,0:2]
    maps = seq[:,:,:,2:4]
    if useCombinedMask:
        loss_mask = seq[:,:,1:,4:5]
        input_seq[:,:,:,0:1] = np.multiply((seq[:,:,:-1,4:5] + 1) // 2, (input_seq[:,:,:,0:1] + 1) // 2) * 2 - 1
    else:
        loss_mask = seq[:,:,1:,1:2]
        input_seq[:,:,:,0:1] = np.multiply((seq[:,:,:-1,4:5] + 1) // 2, (input_seq[:,:,:,0:1] + 1) // 2) * 2 - 1
        seq[:,:,1:,0:1] = np.multiply((seq[:,:,1:,0:1] + 1) // 2, (seq[:,:,1:,3:4] + 1) / 2) * 2 - 1
    target_seq = np.concatenate([seq[:,:,1:,0:1],loss_mask], axis=3)

    tf_matrix = return_transformation[ind][:frame_count-1]
    tf_matrix[:,:,2] = tf_matrix[:,:,2]
    tf_matrix[:,:,5] = - tf_matrix[:,:,5]

    return target_seq, input_seq, maps, tf_matrix

def convert_tfrecord(return_camera_rgb, return_camera_segmentation, return_camera_depth, return_direction, useCombinedMask=False):
    samples = np.arange(samples_per_record)
    shapes = np.repeat(np.array([image_size]), 1, axis=0)
    sequence_steps = np.repeat(np.array([1 + seq_steps * (K + T)]), 1, axis=0)
    combLoss = np.repeat(useCombinedMask, 1, axis=0)
    strname = "imgsze="+str(image_size)+"_fc="+str(seq_length)+"_datasze="+str(data_size[0])+"x"+str(data_size[1])+"_seqlen="+str(seq_steps)+"_K="+str(K)+"_T="+str(T)+"_size="+str(samples_per_record)
    prefixname = prefix + '_' + str(idnr)
    filename = os.path.join(dest_path, prefixname + '_' + strname + '.tfrecord')
    with tf.python_io.TFRecordWriter(filename) as writer:
        for index in samples:
            for f, img_sze, seq, useCM in zip([index], shapes, sequence_steps, combLoss):
                target_seq, input_seq, maps, tf_matrix = load_gridmap_onmove_tfrecord(f, img_sze, seq, useCM)
            seq_batch = target_seq.tostring()
            input_batch = input_seq.tostring()
            map_batch = maps.tostring()
            transformation_batch = tf_matrix.tostring()
            rgb_batch = return_camera_rgb[index].tostring()
            segmentation_batch = return_camera_segmentation[index].tostring()
            depth_batch = return_camera_depth[index].tostring()
            direction_batch = return_direction[index].tostring()
            example = tf.train.Example(
                features=tf.train.Features(
                    feature={
                        'input_seq': _bytes_feature(input_batch),
                        'target_seq': _bytes_feature(seq_batch),
                        'maps': _bytes_feature(map_batch),
                        'tf_matrix': _bytes_feature(transformation_batch),
                        'rgb': _bytes_feature(rgb_batch),
                        'segmentation': _bytes_feature(segmentation_batch),
                        'depth': _bytes_feature(depth_batch),
                        'direction': _bytes_feature(direction_batch)
                    }))
            writer.write(example.SerializeToString())