# Multivariate Timeseries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import plotly.express as px
from sklearn import metrics
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
import tensorflow as tf

def custom_data_prep(dataset, target, start, end, window, horizon):
    x = []
    y = []
    start = start + window
    if end is None:
        end = len(dataset) - horizon
    for i in range(start, end):
        indices = range(i-window, i)
        x.append(dataset[indices])
        indicey = range(i+1, i+1+horizon)
        y.append(target[indicey])
    return np.array(x), np.array(y)

def timeseries_evaluation_metrics_func(y_true, y_pred):
    def mean_absolute_percentage_error(y_true, y_pred):
        y_true, y_pred = np.array(y_true), np.array(y_pred)
        return np.mean(np.abs((y_true - y_pred) / y_true)) * 100
    print('Evaluation metric results:-')
    print(f'MSE is : {metrics.mean_squared_error(y_true, y_pred)}')
    print(f'MAE is : {metrics.mean_absolute_error(y_true, y_pred)}')
    print(
        f'RMSE is : {np.sqrt(metrics.mean_squared_error(y_true, y_pred))}')
    print(f'MAPE is : {mean_absolute_percentage_error(y_true, y_pred)}')
    print(f'R2 is : {metrics.r2_score(y_true, y_pred)}', end='\n\n')

data = pd.read_csv("data.csv", parse_dates=["date"])

print(data.dtypes)

x_scaler = MinMaxScaler()
y_scaler = MinMaxScaler()
x_data = x_scaler.fit_transform(data[["dprice", "nupl"]])
y_data = y_scaler.fit_transform(data[["dprice"]])

hist_window = 30
horizon = 5
train_split = 1500

x_train, y_train = custom_data_prep(
        x_data, y_data, 0, train_split, hist_window, horizon)
x_vali, y_vali = custom_data_prep(
        x_data, y_data, train_split, None, hist_window, horizon)

print('Multiple window of past history')
print(x_train[0])
print('Target horizon')
print(y_train[0])

batch_size = 60
buffer_size = 45
train_data = tf.data.Dataset.from_tensor_slices((x_train, y_train))
train_data = train_data.cache().shuffle(buffer_size).batch(batch_size).repeat()
val_data = tf.data.Dataset.from_tensor_slices((x_vali, y_vali))
val_data = val_data.batch(batch_size).repeat()

lstm_model = tf.keras.models.Sequential([tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(200, return_sequences=True),
                                                                       input_shape=x_train.shape[-2:]),
                                         tf.keras.layers.Dense(
                                         20, activation='tanh'),
                                         tf.keras.layers.Bidirectional(
                                         tf.keras.layers.LSTM(150)),
                                         tf.keras.layers.Dense(
                                         20, activation='tanh'),
                                         tf.keras.layers.Dense(
                                         20, activation='tanh'),
                                         tf.keras.layers.Dropout(0.25),
                                         tf.keras.layers.Dense(
                                            units=horizon),
                                         ])

lstm_model.compile(optimizer='adam', loss='mse')
lstm_model.summary()

model_path = 'Bidirectional_LSTM_Multivariate.h5'
early_stopings = tf.keras.callbacks.EarlyStopping(
        monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='min')
checkpoint = tf.keras.callbacks.ModelCheckpoint(
        model_path, monitor='val_loss', save_best_only=True, mode='min', verbose=0)
callbacks = [early_stopings, checkpoint]
history = lstm_model.fit(train_data, epochs=150, steps_per_epoch=60,
                         validation_data=val_data, validation_steps=30, verbose=1)

plt.figure(figsize=(16, 9))
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train loss', 'validation loss'])
plt.show()

data_val = x_scaler.fit_transform(data[["dprice", "nupl"]].tail(14))
val_rescaled = data_val.reshape(
                             1, data_val.shape[0], data_val.shape[1])
pred = lstm_model.predict(val_rescaled)
pred_Invers = y_scaler.inverse_transform(pred)
print(pred_Invers)

timeseries_evaluation_metrics_func(data_val["dprice"], pred_Invers[0])

plt.figure(figsize=(16, 9))
plt.plot(list(data_val['dprice']))
plt.plot(list(pred_Invers[0]))
plt.title("Actual vs Predicted")
plt.ylabel("BTC Price Change")
plt.legend(('Actual', 'predicted'))
plt.show()


#!/usr/bin/env python
# coding: utf-8

# In[2]:

import math
import time
from IPython.display import clear_output
import matplotlib.pyplot as plt
import pandas as pd

# In[3]:

#Membuat fungsi dari garis (dengan n = 2)
def fungsi_fx(x):
    fx = 2*(x)+12
    return fx

#Golden Search Section
r = (math.sqrt(5)-1) / 2
xl = 4
xu = 10

def golden_search_min(xl,xu,e):
    it = 0
    while e >= 0.05:
        d = r*(xu-xl)
        x1 = xl+d
        x2 = xu-d
        fx1 = fungsi_fx(x1)
        fx2 = fungsi_fx(x2)
        clear_output(wait=True)

        if fx2>fx1:
            xl = x2
            xu = xu
            x2 = x1
            x1 = xl + (r*(xu-xl))
            xopt = x1

        else:
            xl = xl
            xu = x1
            x1 = x2
            x2 = xu - (r*(xu-xl))
            xopt = x2
        it+=1
        print('Iterasi:', it)
        print('xl:', xl)
        print('xu:', xu)
        print('fx optimum:', fungsi_fx(xopt))

        e = ((1-r)*(abs((xu-xl) / xopt))) * 100
        print('Error:', e, '%')

        #Mencari jarak antara dua titik koordinat (dengan n = 2)
        titik_x1 = 4
        titik_y1 = 22
        d_jarak = math.sqrt((titik_x1-xopt)**2 + (titik_y1-fungsi_fx(xopt))**2)
        print('Jarak antara dua titik koordinat:', d_jarak, 'satuan unit')

        time.sleep(3)

golden_search_min(4,10,1)

# In[7]:

import matplotlib.pyplot as plt
import pandas as pd

#Memasukkan data kedalam python
df = pd.read_csv('D:\Tugas Metode Numerik/truck_7.txt', sep = ';')

t = df['time(s)'].tolist()
v = df['velocity(m/s)'].tolist()

#Menghitung jarak yang ditempuh
def trapezoid(x,y,h):
    L = (h/2)*((y[0]+2*(sum(y[1:-1]))+y[-1]))
    return L

S = trapezoid(t,v,2)
print('Jarak yang ditempuh:', S, 'm')

#Mem-plotkan hasil
plt.figure(figsize=(12,8))
plt.plot(t,v,'c')
plt.plot(t,v,'r.')
plt.xlabel('time(s)')
plt.ylabel('speed(m/s)')
plt.show()

# In[ ]:


from datetime import datetime
from pathlib import Path

from scrapy import signals

from services.credentials import prepare_json_credentials, read_json_credentials
from services.logger import log_writer

class OreillySpiderMiddleware:
    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        return None

    def process_spider_output(self, response, result, spider):
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        location = "logs/meta"
        Path(location).mkdir(parents=True, exist_ok=True)
        file_path = f"{location}/{spider.name}.log"
        string = (
            f"ERROR: {str(exception)} occurred for "
            f"{response.request.url} while running {spider.name} spider "
            f"at {datetime.utcnow().isoformat()}\n"
        )
        log_writer(file_path, string)

    def process_start_requests(self, start_requests, spider):
        location = "logs/meta"
        Path(location).mkdir(parents=True, exist_ok=True)

        for r in start_requests:
            file_path = f"{location}/{spider.name}.log"
            string = (
                f"INFO: {r.method} request fired for {spider.name} with {r.url} at {datetime.utcnow().isoformat()}\n"
            )
            log_writer(file_path, string)
            yield r

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)

class OreillyDownloaderMiddleware:
    @classmethod
    def from_crawler(cls, crawler):
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        if request.headers:
            return None

        location = "keys/credentials"
        Path(location).mkdir(parents=True, exist_ok=True)

        if not Path(f"{location}/{spider.name}.sh").exists():
            prepare_json_credentials(spider.name)

        credentials = read_json_credentials(spider.name)

        for key, value in credentials["headers"].items():
            request.headers[key] = value

        for key, value in credentials["cookies"].items():
            request.cookies[key] = value

        request.headers["content-type"] = "application/json"

        if spider.name != "auth":
            credentials = read_json_credentials(spider.name)
            for key, value in credentials["headers"].items():
                if key not in request.headers:
                    request.headers[key] = value
            for key, value in credentials["cookies"].items():
                if key not in request.cookies:
                    request.cookies[key] = value

        return None

    def process_response(self, request, response, spider):
        location = "logs/fails"
        Path(location).mkdir(parents=True, exist_ok=True)

        if (
            response.status == 200
            and response.headers.get("content-type") == "application/json"
            and "success" in response.json()
            and not response.json()["success"]
        ):
            file_path = f"{location}/{spider.name}.log"
            string = f"request to {request.url} failed with {response.json()} at {datetime.utcnow().isoformat()}\n"
            log_writer(file_path, string)

        return response

    def process_exception(self, request, exception, spider):
        location = "logs/fails"
        Path(location).mkdir(parents=True, exist_ok=True)

        file_path = f"{location}/{spider.name}.log"
        string = f"ERROR: request to {request.url} failed with {str(exception)} at {datetime.utcnow().isoformat()}\n"
        log_writer(file_path, string)

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)


import numpy as np
import pandas as pd
from statistics import mode
import distances as distances

class KNeighbors:
    def __init__(self, n_neighbors=3, trainig_data=None, distance_type='euclidean'):
        self.n_neighbors = n_neighbors
        condition = distance_type == 'manhattan'
        self.calc_distance = distances.manhattan_distance if(condition) else distances.euclidean_distance
        self.training_dataframe = trainig_data

    def classify(self,new_object):
        nearest_neighbors = [[float('inf'),None] for _ in range(self.n_neighbors)] # create space to nearest_neighbors
        for _class, _object in self.training_dataframe.iterrows(): # _class = index and _object = data series
            neighbor_distance = self.calc_distance(new_object,_object.values) # get neighbor distance
            nearest_neighbors.sort(key=lambda dist: dist[0], reverse=True) # sort by smallest to largest
            if neighbor_distance < nearest_neighbors[0][0]: # if actual distance < smallest
                nearest_neighbors[0][0] = neighbor_distance # smallest = atual
                nearest_neighbors[0][1] = _class # smallest name = actual class name
        nearest_neighbors = [_class[1] for _class in nearest_neighbors] # set nearest_neighbors
        return mode(nearest_neighbors) # return nearest_neighbors into mode


# imports
import zipfile
from zipfile import ZipFile
import multiprocessing
import os
import sys
import shutil
import json
import requests
import wget
import glob
import pandas as pd
import numpy as np

# Helper Class to load standard COCO-format detection + segmentation data
# Base Class for all datasets
# Subclasses for specific data sources follow
class GetDataset:
    """
    1. Downloads external compressed datasets to local zip directory.
    2. Extracts files from compressed files to local directory.

    Note: default setting extracts files to a subdir of '/content/datasets',
    which is a temporary file location when using Google Colab.
    Make sure to manually enter a save directory if this is not intended behavior.

    # Sample usage with Google Colab and Google Drive:

    # ## Set filepaths:
    archive_base_dir = '/content/drive/MyDrive/ShuffleExchange/datasets'  # Google Drive (long term storage).
    local_base_dir = '/content/datasets'  # Google Colab working directory. (Automatically deleted by Colab upon session exit.)

    # ## Download Data:
    data_loader = Fashionpedia(archive_base_dir=archive_base_dir, local_base_dir=local_base_dir)
    data_loader.get_data(download=True)  # downloads from remote source to archive directory
    data_loader.get_data(unzip=True)  # extracts from archive to working directory

    # ## Load Dataset Info
    df = data_loader.dataframes(subset='valid')  # returns annotations dataframe with mandatory keys 'annotations' and 'image_ path'
    data_loader.citation()  # prints dataset metadata
    """

    def __init__(self, dataset_name, archive_base_dir, local_base_dir,
                 image_source_filepath_dict, annotation_source_filepath_dict):

        self._dataset_name = dataset_name
        self._archive_base_dir = archive_base_dir
        self._local_base_dir = local_base_dir
        self._image_source_filepath_dict = image_source_filepath_dict
        self._annotation_source_filepath_dict = annotation_source_filepath_dict
        self._pad_value = '<PAD>'

        # set save locations
        self._archive_image_dir = os.path.join(self._archive_base_dir, self._dataset_name, 'images')
        self._archive_annotation_dir = os.path.join(self._archive_base_dir, self._dataset_name, 'annotations')
        self._image_dir = os.path.join(self._local_base_dir, self._dataset_name, 'images')
        self._annotations_dir = os.path.join(self._local_base_dir, self._dataset_name, 'annotations')

    def create_folder(self, folder_name):
        if not os.path.exists(folder_name):
            os.makedirs(folder_name)

    def filepath_exist(self, filepath, force_rebuild):
        # check if file was already extracted
        already_exists = os.path.exists(filepath)

        if already_exists and not force_rebuild:
            print(filepath, 'found. Using previously extracted data. (Note: set force_rebuild=True to override)')
            return True
        else:
            return False

    def download_to_archive(self, source_dir, source_filename, subset, datatype, qualifier=None, force_rebuild=False):

        if datatype == 'image':
            destination_dir = os.path.join(self._archive_image_dir, subset)

        elif datatype == 'annotation':
            destination_dir = os.path.join(self._archive_annotation_dir, subset)

        destination_filepath = os.path.join(destination_dir, source_filename)
        source_filepath = os.path.join(source_dir, source_filename)

        # check if file was already downloaded
        exists = self.filepath_exist(destination_filepath, force_rebuild)

        if not exists:
            # create destination directory structure
            self.create_folder(destination_dir)

            # overwrite any existing file
            if os.path.exists(destination_filepath):
                os.remove(destination_filepath)
            wget.download(source_filepath, destination_filepath)

            downloaded = True

        else:
            downloaded = False

        return downloaded

    def download_to_local_dir(self, source_file, datatype, subset, force_rebuild):

        if datatype == 'image':
            destination_dir = os.path.join(self._image_dir, subset)
        elif datatype == 'annotation':
            destination_dir = os.path.join(self._annotations_dir, subset)
        else:
            print('must set datatype = image or annotation')

        filename = os.path.split(source_file)[1]
        destination_path = os.path.join(destination_dir, filename)
        # check if file was already downloaded
        exists = self.filepath_exist(destination_path, force_rebuild)
        if exists:
            downloaded = False

        else: # create local directory and copy file
            self.create_folder(destination_dir)
            shutil.copyfile(source_file, destination_path)
            downloaded = True

        return downloaded

    def unzip_to_local_dir(self, source_file, datatype, subset, force_rebuild):

        if datatype == 'image':
            destination_dir = os.path.join(self._image_dir, subset)

        elif datatype == 'annotation':
            destination_dir = os.path.join(self._annotations_dir, subset)

        exists = self.filepath_exist(destination_dir, force_rebuild)
        if exists:
            extracted = False

        else:
            # create local directory
            if not os.path.exists(destination_dir):
                self.create_folder(destination_dir)

            # extract file and return to working directory
            cwd = os.getcwd()

            with zipfile.ZipFile(source_file, 'r') as zipped:
                zipped.extractall(destination_dir)
            extracted = True

        return extracted

    def get_data(self, download=False, unzip=False, force_rebuild=False):
        """ downloads to archive and extracts to local dir """

        base_dir = self._image_source_filepath_dict['base_dir']

        if download:
            for subset in ['train', 'test']:

                # images
                base_dir = self._image_source_filepath_dict['base_dir']
                qualifier = self._image_source_filepath_dict['qualifier']

                for filename in self._image_source_filepath_dict[subset]:

                    print('downloading:', filename)
                    self.download_to_archive(base_dir, filename, subset, datatype='image',
                                            qualifier=qualifier, force_rebuild=force_rebuild)

                # annotations
                qualifier = self._image_source_filepath_dict['qualifier']
                base_dir = self._annotation_source_filepath_dict['base_dir']

                for filename in self._annotation_source_filepath_dict[subset]:
                    print('downloading:', filename)
                    self.download_to_archive(base_dir, filename, subset, datatype='annotation',
                                            qualifier=qualifier, force_rebuild=force_rebuild)

        if unzip:
            for subset in ['train', 'test']:

                # get images
                base_image_path = os.path.join(self._archive_image_dir, subset)

                if os.path.exists(base_image_path):
                    image_filepath_list = os.listdir(base_image_path)

                    for source_file in list(reversed(image_filepath_list)):
                        print('extracting:', source_file)

                        datatype = 'image'
                        source_filepath = os.path.join(base_image_path, source_file)

                        if zipfile.is_zipfile(source_filepath):
                            self.unzip_to_local_dir(source_filepath, datatype, subset, force_rebuild)
                        else:
                            self.download_to_local_dir(source_filepath, datatype, subset, force_rebuild)

                # get annotations
                base_annotation_path = os.path.join(self._archive_annotation_dir, subset)
                if os.path.exists(base_annotation_path):
                    annotation_filepath_list = os.listdir(base_annotation_path)

                    for source_file in annotation_filepath_list:
                        datatype = 'annotation'
                        source_filepath = os.path.join(base_annotation_path, source_file)

                        # copy / extract
                        if zipfile.is_zipfile(source_filepath):
                            self.unzip_to_local_dir(source_filepath, datatype, subset, force_rebuild)
                        else:
                            self.download_to_local_dir(source_filepath, datatype, subset, force_rebuild)

        return None

class COCOStandard(GetDataset):

    def __init__(self, local_base_dir=None, archive_base_dir=None, **kwargs):

        dataset_name = 'COCO'

        # Location of Source Files
        external_base_dir = 'http://images.cocodataset.org'

        image_source_filepath_dict = {
            'base_dir': os.path.join(external_base_dir, 'zips'),
            'qualifier': {'name': None,},
            'train': ['train2017.zip'],
            'test': ['val2017.zip']#['test2017.zip', 'val2017.zip']
        }

        annotation_source_filepath_dict = {
            'base_dir': os.path.join(external_base_dir, 'annotations'),
            'qualifier': {'name': None,},
            'train': ['stuff_annotations_trainval2017.zip',
                      'panoptic_annotations_trainval2017.zip',
                      'annotations_trainval2017.zip'],
            'test': []}  # train list includes both validation data

        # initialize class
        super().__init__(dataset_name, archive_base_dir, local_base_dir,
                          image_source_filepath_dict, annotation_source_filepath_dict)

        # placeholder attributes (filled by later function calls)
        self._prepared_info = None
        self._category_indx_to_name = None
        self._attribute_indx_to_name = None

    def prepare_COCO_from_json(self, subset, force_rebuild=False):

        if subset == 'train':
            image_dir = '/content/COCO/images/train/train2017'
            json_filenames = ['/content/COCO/annotations/train/annotations/instances_train2017.json',
                              '/content/COCO/annotations/train/annotations/person_keypoints_train2017.json',
                              '/content/COCO/annotations/train/annotations/captions_train2017.json'
                              ]

        elif subset == 'val':
            image_dir = '/content/COCO/images/test/val2017'
            json_filenames = ['/content/COCO/annotations/train/annotations/instances_val2017.json',
                              '/content/COCO/annotations/train/annotations/person_keypoints_val2017.json',
                              '/content/COCO/annotations/train/annotations/captions_val2017.json'
                             ]

        instance_info = self.load_COCO_json(json_filenames[0])
        #keypoint_info = self.load_COCO_json(json_filenames[1]) # not yet coded to handle multiple sets
        #caption_info = self.load_COCO_json(json_filenames[2]) ####################### not yet appropriately coded for this file

        # create or load annotations df
        # check for previously prepared ds
        possible_filepath = (os.path.join(self._archive_annotation_dir, f'{subset}_combined_annotations_df.json'))

        if os.path.exists(possible_filepath) and not force_rebuild:
            combined_annotations_df = pd.read_json(possible_filepath)

        else:  # create df from scratch
            combined_annotations_df = self.create_combined_COCO_detections_df(
                        instance_info['images_df'], instance_info['annotations_df'])

            combined_annotations_df['image_path'] = \
                combined_annotations_df['file_name'].apply(lambda x: os.path.join(image_dir, x))

            # update index to reclaim 'image_id' column
            combined_annotations_df = combined_annotations_df.reset_index().rename(columns={'image_id':'id_num'})

            # save to json
            combined_annotations_df.to_json(os.path.join(self._archive_annotation_dir,
                                                     f'{subset}_combined_annotations_df.json'))

        self._prepared_info = {'annotations_df': combined_annotations_df,
                               'categories_df': instance_info['categories_df'],
                               'meta_info': instance_info['meta_info']}

        # Display info
        print('Returns dictionary with keys:', self._prepared_info.keys())
        print()
        print('annotations_df:', self._prepared_info['annotations_df'].columns)
        print()
        print('note: bbox provided in normalized COCO format: [xmin, ymin, width, height]')

        return self._prepared_info

    def load_COCO_json(self, filename):

        # load json annotation files
        with open(filename) as f:
            all_info = json.load(f)

        meta_info = all_info['info']

        # images
        images_df = pd.DataFrame(all_info['images'])
        images_df = images_df.set_index('id')

        # annotations
        annotations_df = pd.DataFrame(all_info['annotations'])
        annotations_df = annotations_df.set_index('image_id')

        # category names
        if 'categories' in all_info.keys():
            categories_df = pd.DataFrame(all_info['categories'])
            self._category_indx_to_name = categories_df.set_index('id')['name'].to_dict()
        else:
            categories_df = None

        # attribute names (Fashionpedia dataset)
        if 'attributes' in all_info.keys():
            attribute_df = pd.DataFrame(all_info['attributes'])
            self._attribute_indx_to_name = attribute_df.set_index('id')['name'].to_dict()
        else:
            categories_df = None

        return {'categories_df': categories_df, 'annotations_df': annotations_df,
                'images_df': images_df, 'meta_info': meta_info}

    def create_combined_COCO_detections_df(self, images_df, annotations_df):

        # prepare annotations dictionary
        # begin with columns included in all sets' images_df
        combined_annotations_dict = {'image_id': [], 'width':[], 'height':[],
                                     'coco_url':[], 'file_name':[],}

        # object count
        if 'bbox' in annotations_df.columns:  # not used in captions
            combined_annotations_dict['num_boxes'] = []
            combined_annotations_dict['bbox'] = []

        # use strings instead of integer category labels
        if 'category_id' in annotations_df.columns:  # not used in captions
            def cat_name(cat_id):
                return self._category_indx_to_name[cat_id]

            combined_annotations_dict['category'] = []
            annotations_df['category_id'] = annotations_df['category_id'].apply(cat_name)

        # use strings instead of integer attribute labels
        if 'attribute_ids' in annotations_df.columns:  # used in Fashionpedia
            def att_name(att_id_list):
                return [self._attribute_indx_to_name[att_id] for att_id in att_id_list]

            combined_annotations_dict['attribute'] = []
            annotations_df['attribute'] = annotations_df['attribute'].apply(att_name)

        for keyname in annotations_df.columns:  # handles other common formats such as keypoints
            if keyname in ['id', 'bbox', 'category_id', 'attribute_ids']:
                continue
            combined_annotations_dict[keyname] = []

        # function to consolidate object values from individual column
        def gather_objects(keyname, location, num_boxes):
            if num_boxes > 1:
                combined_info = []
                for info in annotations_df.loc[location][keyname]:
                    combined_info.append([info])
            else:
                combined_info = [[annotations_df.loc[location][keyname]]]
            return combined_info

        # function to consolidate all object values for single image
        def combine_annotation_info(location):

            # image ID
            combined_annotations_dict['image_id'].append(location)

            # image dims
            width = images_df.loc[location]['width']
            height = images_df.loc[location]['height']
            combined_annotations_dict['width'].append(width)
            combined_annotations_dict['height'].append(height)

            # file name
            file_name = images_df.loc[location]['file_name']
            combined_annotations_dict['file_name'].append(file_name)

            # url
            coco_url = images_df.loc[location]['coco_url']
            combined_annotations_dict['coco_url'].append(coco_url)

            # Object Detection
            # bboxes & number of bboxes
            if 'bbox' in annotations_df.columns:

                # ## record number of boxes
                num_boxes = 0
                combined_bbox = []

                if type(annotations_df.loc[location]['bbox']) is list:
                    combined_bbox = annotations_df.loc[location]['bbox']
                    num_boxes = 1
                else:
                    for bbox in annotations_df.loc[location]['bbox']:
                        combined_bbox.append(bbox)
                        num_boxes += 1

                combined_annotations_dict['num_boxes'].append(num_boxes)

                # ## add normalized bbox
                combined_bbox = np.array(combined_bbox) / np.array([[width, height, width, height]])
                combined_bbox = combined_bbox.tolist()
                combined_annotations_dict['bbox'].append(combined_bbox)

            # category
            if 'category_id' in annotations_df.columns:
                combined_category_id = gather_objects('category_id', location, num_boxes)
                combined_annotations_dict['category'].append(combined_category_id)

            # area
            if 'area' in annotations_df.columns:
                combined_area = gather_objects('area', location, num_boxes)
                combined_annotations_dict['area'].append(combined_area)

            # iscrowd
            if 'iscrowd' in annotations_df.columns:
                if num_boxes > 1:
                    iscrowd = annotations_df.loc[location]['iscrowd'].iloc[0]  # only one unique value per image
                else:
                    iscrowd = annotations_df.loc[location]['iscrowd']
                combined_annotations_dict['iscrowd'].append(iscrowd)

            # Optional Components
            # Segmentation
            if 'segmentation' in annotations_df.columns:

                if type(annotations_df.loc[location]['segmentation']) is list:
                    combined_segmentation = [annotations_df.loc[location]['segmentation']]

                else:
                    combined_segmentation = []
                    for segmentation in annotations_df.loc[location]['segmentation']:
                        combined_segmentation.append(segmentation)

                combined_annotations_dict['segmentation'].append(combined_segmentation)

            if 'segments_info' in annotations_df.columns:
                combined_segments_info = gather_objects('segments_info', location, num_boxes)
                combined_annotations_dict['segments_info'].append(combined_segments_info)

            # Keypoints
            if 'num_keypoints' in annotations_df.columns:
                num_keypoints = annotations_df.loc[location]['num_keypoints']
                combined_annotations_dict['num_keypoints'].append(num_keypoints)

            if 'keypoints' in annotations_df.columns:

                if type(annotations_df.loc[location]['keypoints']) is list:
                    combined_keypoints = [annotations_df.loc[location]['keypoints']]

                else:
                    combined_keypoints = []
                    for keypoints in annotations_df.loc[location]['keypoints']:
                        combined_keypoints.append(keypoints)

                combined_annotations_dict['keypoints'].append(combined_keypoints)

            # Captioning
            if 'caption' in annotations_df.columns:
                caption = annotations_df.loc[location]['caption']
                combined_annotations_dict['caption'].append(caption)

            # DensePose
            for keyname in ['dp_I', 'dp_U', 'dp_V', 'dp_x', 'dp_y', 'dp_masks']:
                if keyname in annotations_df.columns:

                    combined_info = []

                    for keyname_info in annotations_df.loc[location][keyname]:
                        combined_info.append(keyname_info)

                    combined_annotations_dict[keyname].append(combined_info)

            # Other Annotations (not in standard COCO sets)
            if 'attribute_ids' in annotations_df.columns:
                combined_info = []

                for attribute_info in annotations_df.loc[location]['attribute_ids']:
                    combined_info.append(attribute_info)

                combined_annotations_dict['attribute'].append(combined_info)

            return None

        # Add each prepared image info to dict
        all_image_ids = list(annotations_df.index.unique())

        for image_id in all_image_ids:
            combine_annotation_info(image_id)

        # create dataframe indexed on 'image_id' (index for join)
        combined_annotations_df = pd.DataFrame(combined_annotations_dict)
        combined_annotations_df = combined_annotations_df.set_index('image_id')

        return combined_annotations_df

    # helper functions
    def max_num_obj(self):
        return self._prepared_info['annotations_df']['bbox'].apply(lambda x: len(x)).max()

    def dataframes(self):
        return self._prepared_info['annotations_df']

    def get_vocab(self):

        vocab = {'category': list(self._category_indx_to_name.values())}

        if self._attribute_indx_to_name:
            vocab['attribute'] = list(self._attribute_indx_to_name.values())

        return vocab

class Fashionpedia(GetDataset):
    """
    Loads Fashionpedia Dataset, https://fashionpedia.github.io/home/index.html.
    """

    def __init__(self, local_base_dir=None, archive_base_dir=None, **kwargs):

        self._dataset_name = 'Fashionpedia'
        self._pad_value = '<PAD>'

        # Location of Source Files
        external_base_dir = 'https://s3.amazonaws.com/ifashionist-dataset/'

        image_source_filepath_dict = {
            'base_dir': os.path.join(external_base_dir, 'images'),

            'qualifier': {'name': None,},
            'train': ['train2020.zip'],
            'test': ['val_test2020.zip']
        }

        annotation_source_filepath_dict = {
            'base_dir': os.path.join(external_base_dir, 'annotations'),
            'qualifier': {'name': None,},
            'train': ['instances_attributes_train2020.json'],
                      #'attributes_train2020.json'],
            'test': ['instances_attributes_val2020.json']
                     #'attributes_val2020.json']
        }

        # initialize class
        super().__init__(self._dataset_name, archive_base_dir, local_base_dir,
                          image_source_filepath_dict, annotation_source_filepath_dict)

        # placeholder attributes
        self._id_to_category = None
        self._id_to_attribute = None
        self._attribute_vocab = None
        self._category_vocab = None

    # CITATION
    def citation(self):
        print('''Dataset provided with Creative Commons Attribution 4.0 License and is publicaly available at https://arxiv.org/pdf/1905.12794.pdf.
        CITATION: @inproceedings{jia2020fashionpedia,
            title={Fashionpedia: Ontology, Segmentation, and an Attribute Localization Dataset},
            author={Jia, Menglin and Shi, Mengyun and Sirotenko, Mikhail and Cui, Yin and Cardie, Claire and Hariharan, Bharath and Adam, Hartwig and Belongie, Serge}
            booktitle={European Conference on Computer Vision (ECCV)},
            year={2020}
            }''')

    # NOTES
    def notes(self):
        bbox_note = 'bbox provided in normalized COCO format: [xmin, ymin, width, height]'
        segmentation_note = 'segmentations not normalized and contain mix of polyhons and rle formats'
        print(bbox_note, '\n',segmentation_note)

        return 'normalized COCO'

    # json parsing function
    def json_to_dataframe(self, filename, save_subset):

        # load json annotation files
        with open(filename) as f:
            all_info = json.load(f)

        print(all_info.keys())
        # get index --> category maps
        category_df = pd.DataFrame(all_info['categories'])
        category_df = category_df.set_index('id').rename(columns={'name':'category'})
        id_to_category = category_df['category'].to_dict()

        # get attribute <--> index maps
        attribute_df = pd.DataFrame(all_info['attributes'])
        attribute_df = attribute_df.set_index('id').rename(columns={'name':'attribute'})
        id_to_attribute = attribute_df['attribute'].to_dict()

        dummy_indx = -1  # add NONE placeholder value
        id_to_attribute[dummy_indx] = self._pad_value

        # get box and class annotations
        df_image = pd.DataFrame(all_info['images']).set_index('id')
        df_annotations = pd.DataFrame(all_info['annotations']).set_index('image_id')

        # ## update category_id list nesting to match attribute_ids
        df_annotations['category_id'] = df_annotations['category_id'].apply(lambda x: [x])

        # Collect each image's annotations as a dictionary of lists
        images_dict = {}
        all_ids = df_image.index

        for id_num in all_ids:

            # restrict data to the specific image
            image_info_i = df_image.loc[id_num]
            annotation_info_i = df_annotations.loc[id_num]

            # get image meta info
            file_name = image_info_i['file_name']
            image_width = image_info_i['width']
            image_height = image_info_i['height']
            num_boxes = len(annotation_info_i)

            # collect box and annotation info.
            try:
                annotation_info_i.columns  # verifies this is a dataframe
                category_list = annotation_info_i['category_id'].to_list()
                attribute_list = annotation_info_i['attribute_ids'].to_list()
                segmentation_list = annotation_info_i['segmentation'].to_list()
                box_vals = np.array(annotation_info_i['bbox'].to_list())

            except:  # entry is a series, there is only one object
                num_boxes = 1
                category_list = [annotation_info_i['category_id']]
                attribute_list = [annotation_info_i['attribute_ids']]
                segmentation_list = [annotation_info_i['segmentation']]
                box_vals = np.array([annotation_info_i['bbox']])

            # replace category index with strings
            category_list = [[id_to_category[indx]
                               for indx in cat] for cat in category_list]

            # pad missing attributes with <NONE> class and replace index with strings
            for i in range(num_boxes):
                if attribute_list[i] == []:
                    attribute_list[i] = [dummy_indx]
            attribute_list = [[id_to_attribute[indx]
                               for indx in att] for att in attribute_list]

            # normalize box values into [0,1]
            box_vals = box_vals / np.array([image_width, image_height, image_width, image_height])
            box_vals = box_vals.tolist()

            # save image results
            images_dict[id_num] = {'file_name': file_name,
                                   'bbox': box_vals,
                                   'category': category_list,
                                   'attribute': attribute_list,
                                   'segmentation': segmentation_list,
                                   'image_width': image_width,
                                   'image_height': image_height,
                                   'num_boxes': num_boxes}

        # combine results and reclaim index column
        full_annotations_df = pd.DataFrame.from_dict(images_dict, orient='index')
        full_annotations_df = full_annotations_df.reset_index().rename(columns={'index':'id_num'})

        # save to json
        save_to_archive_path = os.path.join(self._archive_annotation_dir, save_subset, save_subset + '_full_annotations_df.json')
        full_annotations_df.to_json(save_to_archive_path)

        return full_annotations_df

    def dataframes(self, subset='train', force_rebuild=False):

        if subset not in ['train', 'val']:
            print('Error: subset must be either "train" or "val"')
            return None

        # select annotations file
        if subset=='train':
            save_subset = 'train'
        elif subset == 'val':
            save_subset = 'test'  # rename for updating image paths

        base_annotation_path = os.path.join(self._annotations_dir, save_subset)
        annotation_file_list = self._annotation_source_filepath_dict[save_subset]

        for filename in annotation_file_list:
            json_file = os.path.join(base_annotation_path, filename)

        save_to_archive_path = os.path.join(self._archive_annotation_dir, save_subset, save_subset + '_full_annotations_df.json')

        # check dataset was previously prepared
        already_exists = self.filepath_exist(save_to_archive_path, force_rebuild)

        if already_exists:
            print('Previously prepared datafile found. Loading from saved file.')
            print('Use "force_rebuild=True" to generate dataframe from scratch')
            full_annotations_df = pd.read_json(save_to_archive_path)

        else:
            full_annotations_df = self.json_to_dataframe(json_file, save_subset)

        # add image path info
        directory = os.path.join(self._image_dir, save_subset)
        full_annotations_df['image_path'] = full_annotations_df['file_name'].apply(lambda x: os.path.join(directory, save_subset, x))

        # save as attribute
        self._full_annotations_df = full_annotations_df

        return full_annotations_df

    # helper functions
    def max_num_obj(self):
        return self._full_annotations_df['bbox'].apply(lambda x: len(x)).max()

    def get_vocab(self):

        category_vocab = {elem for cat_list in self._full_annotations_df['category'].to_list()
                                for cat in cat_list for elem in cat}

        attribute_vocab = {elem for atts_list in self._full_annotations_df['attribute'].to_list()
                                 for att in atts_list for elem in att}

        vocab = {'category': list(category_vocab),
                 'attribute': list(attribute_vocab)
                 }
        return vocab

class UnsplashLite(GetDataset):
    """
    Note: this utilizes Pandas' API, so follows their file name/structure conventions.

    Loads Unsplash.com's public use "lite" dataset containing 25,000 images
    with metadata including keywords and colors.
    Call self.citation() to print info.  As of writing, this dataset was
    made available for commercial usage subject to the terms at
    https://github.com/unsplash/datasets/blob/master/TERMS.md

    Sample usage:
    data_loader = UnsplashLite()
    data_loader.get_data(download=False, unzip=False)  # get files
    unsplash_df_dict = data_loader.dataframes()  # create dataframes
    unsplash_df_dict['keywords']['keyword'].unique()  # get all keywords
    """

    def __init__(self, local_base_dir=None, archive_base_dir=None, **kwargs):

        self._dataset_name = 'UnsplashLite'

        # Location of Source Files
        external_base_dir = 'https://unsplash.com/data/lite/'

        image_source_filepath_dict = {
            'base_dir': external_base_dir,
            'qualifier': {'name': None,},
            'train': ['latest'],
            'test': None
        }

        annotation_source_filepath_dict = {
            'base_dir': os.path.join(external_base_dir, 'annotations'),
            'qualifier': {'name': None,},
            'train': None,
            'test': None,
        }

        # initialize class
        super().__init__(self._dataset_name, archive_base_dir, local_base_dir,
                          image_source_filepath_dict, annotation_source_filepath_dict)

    # CITATION
    def citation(self):
        print('''CITATION: Unsplash Lite Dataset 1.2.0, unsplash.com/data.
        Lite dataset: available for commercial and noncommercial usage, containing 25k nature-themed Unsplash photos, 25k keywords, and 1M searches.
        See https://github.com/unsplash/datasets for info and
        https://github.com/unsplash/datasets/blob/master/TERMS.md for usage rights''')

    def dataframes(self):
        """ This code is from the Unsplash.com API.
        Return dictionary containing pandas dataframes with names of the form:
        datasets('photos'), datasets('keywords'), datasets('collections'),
        datasets('conversions'), datasets('colors').
        See DOCS.md in the unzip dir for details.
         """

        # go to root dir
        #os.chdir('/')
        #save_dir = self.local_zip_extract_dir_dict['clothing_dataset_grigorev']
        #csv_file = os.path.join(save_dir, 'images.csv')

        documents = ['photos', 'keywords', 'collections', 'conversions', 'colors']
        datasets = {}

        for doc in documents:
            files = glob.glob(os.path.join(self._archive_image_dir, doc) + ".tsv*")

            subsets = []
            for filename in files:
                df = pd.read_csv(filename, sep='\t', header=0)
                subsets.append(df)

            datasets[doc] = pd.concat(subsets, axis=0, ignore_index=True)

        print(f'This returns a dictionary of dataframes. \
        Dict Keys: {documents}')

        return datasets

class ClothingDatasetGrigorev(GetDataset):
    """
    Loads Alexey Grigorev's Clothing Dataset, from Github repository.
    Dataset was made available for commercial purposes at time of this writing.
    See https://github.com/alexeygrigorev/clothing-dataset#readme for acknowledgements.

    Sample usage:
    data_loader = ClothingDatasetGrigorev()
    data_loader.get_data(download=False, unzip=False)  # get files
    df_dict = data_loader.dataframes()  # create dataframes
    """

    def __init__(self, local_base_dir=None, archive_base_dir=None, **kwargs):

        name = 'ClothingDatasetGrigorev'
        external_source_dict = {'clothing_dataset_grigorev': 'https://github.com/alexeygrigorev/clothing-dataset.git'}
        archive_save_dir_dict = {'clothing_dataset_grigorev': None}  # get_data() overrident for Github download

        local_zip_extract_dir_dict = {'clothing_dataset_grigorev': name}
        local_copy_dir_dict = {'clothing_dataset_grigorev': name}

        # initialize class
        super().__init__(local_base_dir=local_base_dir,
                         archive_base_dir=archive_base_dir,
                         external_source_dict=external_source_dict,
                         archive_save_dir_dict=archive_save_dir_dict,
                         local_zip_extract_dir_dict=local_zip_extract_dir_dict,
                         local_copy_dir_dict=local_copy_dir_dict,
                         name=name)

    # CITATION
    def citation(self):
        print('''CITATION: Alexey Grigorev's Clothing Dataset
            Github: https://github.com/alexeygrigorev/clothing-dataset
            Medium Post: https://medium.com/data-science-insider/clothing-dataset-5b72cd7c3f1f
            Full resolution photos: https://www.kaggle.com/agrigorev/clothing-dataset-full/''')

    def get_data(self, **kwargs):
        """ Gets dataset by cloning Github repo. Overrides base class def. """

        source = self.external_source_dict['clothing_dataset_grigorev']
        destination = self.local_zip_extract_dir_dict['clothing_dataset_grigorev']

       # clone repo and return to working dir
        os.chdir('/')
        os.system(f'git clone {source} {destination}')
        os.chdir(self.working_dir)

        return None

    def dataframes(self):
        """ Returns one dataframe conatining labels and image names """

        # go to root dir
        os.chdir('/')
        save_dir = self.local_zip_extract_dir_dict['clothing_dataset_grigorev']
        csv_file = os.path.join(save_dir, 'images.csv')

        # create dataframe & some cleanup
        df = pd.read_csv(csv_file)
        df = df.drop(columns=['sender_id', 'kids'])
        df = df[df['label']!= 'Not sure']

        # add image path info
        df['image_path'] = df['image'].apply(lambda x: os.path.join(save_dir, 'images', x))

        return df


import io
import tempfile
from unittest import mock

from PIL import Image
from django.core.files import File
from django.core.files.base import ContentFile
from django.test import TestCase, Client, override_settings
from django.urls import reverse
from posts.models import Post, Group, User
from django.core.cache import cache

class PostTests(TestCase):
    def setUp(self):
        cache.clear()
        self.no_auth_client = Client()
        self.auth_client = Client()
        self.auth_client2 = Client()
        self.auth_client3 = Client()
        self.user = User.objects.create_user(
            username='testuser',
            email='test@test.ru',
        )
        self.user2 = User.objects.create_user(
            username='testuser2',
            email='test2@test.ru',
        )
        self.user3 = User.objects.create_user(
            username='testuser3',
            email='test3@test.ru',
        )
        self.group = Group.objects.create(
            title='test',
            slug='test_group',
            description='empty'
        )
        self.auth_client.force_login(self.user)
        self.auth_client2.force_login(self.user2)
        self.auth_client3.force_login(self.user3)

    def create_post(self):
        return Post.objects.create(
            text='test text new post',
            author=self.user,
            group=self.group)

    def create_post_with_image(self, image):
        return Post.objects.create(
            text='test text new post with image',
            author=self.user,
            group=self.group,
            image=image)

    def check_text_in_url(self, text):
        author = self.post.author
        group = self.post.group
        for url in (
            reverse('index'),
            reverse('profile', kwargs={'username': self.user.username}),
            reverse('post_view', kwargs={'username': self.user.username,
                                         'post_id': self.post.id})):
            with self.subTest(url=url):
                response = self.auth_client.get(url)
                #  проверку паджинатора убрал - его нет на странице с постом
                self.assertContains(response, text),
                self.assertEqual(response.context['post'].author, author),
                self.assertEqual(response.context['post'].group, group)

    def test_profile(self):
        response = self.no_auth_client.get(
            reverse('profile', kwargs={'username': self.user.username}))
        self.assertEqual(response.status_code, 200)

    def test_no_auth_new_post(self):
        response = self.no_auth_client.post(
            reverse('new_post'), data={'text': 'test text'})
        self.assertNotEqual(response.status_code, 200)
        self.assertRedirects(response, '/auth/login/?next=/new')

    def test_auth_new_post(self):
        response = self.auth_client.post(
            reverse('new_post'),
            data={'text': 'test text new post', 'group': self.group.id},
            follow=True
        )
        self.assertEqual(response.status_code, 200)
        self.assertTrue(Post.objects.filter(
            text='test text new post', group=self.group.id).exists()
        )

    def test_view_auth_post(self):
        self.post = self.create_post()
        self.check_text_in_url('test text new post')

    def test_edit_auth_post(self):
        self.post = self.create_post()
        response = self.auth_client.get(reverse('post_edit', kwargs={
            'username': self.user.username, 'post_id': self.post.id}))
        self.assertEqual(response.status_code, 200)
        self.post.text = 'test edited text'
        self.post.save()
        self.check_text_in_url('test edited text')
        self.assertTrue(Post.objects.filter(
            text='test edited text',
            group=self.group.id,
            author=self.user).exists())

    def test_image_in_urls(self):
        self.post = self.create_post_with_image('test_img.png')
        self.check_text_in_url('<img')

    def test_post_with_image(self):
        with tempfile.TemporaryDirectory() as temp_directory:
            with override_settings(MEDIA_ROOT=temp_directory):
                byte_img = io.BytesIO()
                im = Image.new("RGB", size=(500, 500), color=(255, 0, 0, 0))
                im.save(byte_img, format='png')
                byte_img.seek(0)
                response = self.auth_client.post(
                    reverse('new_post'),
                    data={'text': 'image',
                          'group': self.group.id,
                          'image': ContentFile(byte_img.read(), name='t.png')},
                    follow=True
                )
                self.assertEqual(response.status_code, 200)
                self.assertContains(response, '<img')
                self.assertEqual(Post.objects.count(), 1)

    def test_not_image_upload(self):
        error_text = (
            'Загрузите правильное изображение.'
            ' Файл, который вы загрузили, поврежден'
            ' или не является изображением.'
        )
        file_mock = mock.MagicMock(spec=File, name='test.txt')
        response = self.auth_client.post(
            reverse('new_post'),
            data={'text': 'image', 'group': self.group.id, 'image': file_mock},
            follow=True
        )
        self.assertFormError(response, form='form', field='image',
                             errors=error_text)
        self.assertEqual(Post.objects.count(), 0)

    def test_cache_index(self):
        self.auth_client.get(reverse('index'))
        self.auth_client.post(
            reverse('new_post'),
            data={'text': 'test cache', 'group': self.group.id},
            follow=True)
        response = self.auth_client.get(reverse('index'))
        self.assertNotContains(response, 'test cache')

    def test_auth_follow(self):
        self.assertEqual(self.user.follower.count(), 0)
        self.auth_client.get(reverse('profile_follow',
                                     kwargs={'username': self.user2.username}))
        self.assertEqual(self.user.follower.count(), 1)

    def test_auth_unfollow(self):
        self.auth_client.get(reverse('profile_follow',
                                     kwargs={'username': self.user2.username}))
        self.assertEqual(self.user.follower.count(), 1)
        self.auth_client.get(reverse('profile_unfollow',
                                     kwargs={'username': self.user2.username}))
        self.assertEqual(self.user.follower.count(), 0)

    def test_add_comment(self):
        post = self.create_post()
        self.auth_client.post(reverse('add_comment',
                                      kwargs={'username': post.author.username,
                                              'post_id': post.id}),
                              data={'post': post,
                                    'author': post.author.username,
                                    'text': 'new test comment'}, follow=True)
        self.assertEqual(post.comments.count(), 1)

    def test_no_auth_add_comment(self):
        post = self.create_post()
        response = self.no_auth_client.post(
            reverse('add_comment', kwargs={'username': post.author.username,
                                           'post_id': post.id}),
            data={'post': post, 'author': post.author.username,
                  'text': 'new test comment'})
        self.assertEqual(post.comments.count(), 0)
        self.assertEqual(response.status_code, 302)
        self.assertRedirects(
            response,
            f'/auth/login/?next=/{post.author.username}/{post.id}/comment')

    def test_follow_post(self):
        self.create_post()
        self.auth_client2.get(reverse(
            'profile_follow', kwargs={'username': self.user.username}))
        response = self.auth_client.get(reverse('follow_index'))
        response2 = self.auth_client2.get(reverse('follow_index'))
        response3 = self.auth_client3.get(reverse('follow_index'))
        self.assertNotContains(response, 'test text new post')
        self.assertContains(response2, 'test text new post')
        self.assertNotContains(response3, 'test text new post')

class ServerErrorsTest(TestCase):
    def setUp(self):
        self.client = Client()

    def test_404(self):
        response = self.client.get('/nohaveadminpage/posts')
        self.assertEqual(response.status_code, 404)


import math as m
import numpy as np
import matplotlib.pyplot as plt
import json

with open("./NEAT_RUN_STATS/silverstone-base-run-1.json", 'r') as f:
    baseline_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-base-run-2.json", 'r') as f:
    baseline_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-base-run-3.json", 'r') as f:
    baseline_run3 = json.load(f)

generations = baseline_run1['generations'][:20]
baseline_run1_popmax = np.array(baseline_run1['best_fitness'][:20])
baseline_run2_popmax = np.array(baseline_run2['best_fitness'][:20])
baseline_run3_popmax = np.array(baseline_run3['best_fitness'][:20])
baseline_runs_popmaxes = np.vstack((baseline_run1_popmax, baseline_run2_popmax, baseline_run3_popmax))
baseline_means = baseline_runs_popmaxes.mean(axis=0)
baseline_stds = baseline_runs_popmaxes.std(axis=0, ddof=1)

with open("./NEAT_RUN_STATS/silverstone-4species-run-1.json", 'r') as f:
    species4_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-4species-run-2.json", 'r') as f:
    species4_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-4species-run-3.json", 'r') as f:
    species4_run3 = json.load(f)

# generations = baseline_run1['generations']
species4_run1_popmax = np.array(species4_run1['best_fitness'])
species4_run2_popmax = np.array(species4_run2['best_fitness'])
species4_run3_popmax = np.array(species4_run3['best_fitness'])
species4_runs_popmaxes = np.vstack((species4_run1_popmax, species4_run2_popmax, species4_run3_popmax))
species4_means = species4_runs_popmaxes.mean(axis=0)
species4_stds = species4_runs_popmaxes.std(axis=0, ddof=1)

with open("./NEAT_RUN_STATS/silverstone-8species-run-1.json", 'r') as f:
    species8_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-8species-run-2.json", 'r') as f:
    species8_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-8species-run-3.json", 'r') as f:
    species8_run3 = json.load(f)

# generations = baseline_run1['generations']
species8_run1_popmax = np.array(species8_run1['best_fitness'])
species8_run2_popmax = np.array(species8_run2['best_fitness'])
species8_run3_popmax = np.array(species8_run3['best_fitness'])
species8_runs_popmaxes = np.vstack((species8_run1_popmax, species8_run2_popmax, species8_run3_popmax))
species8_means = species8_runs_popmaxes.mean(axis=0)
species8_stds = species8_runs_popmaxes.std(axis=0, ddof=1)

with open("./NEAT_RUN_STATS/silverstone-30pop-run-1.json", 'r') as f:
    pop30_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-30pop-run-2.json", 'r') as f:
    pop30_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-30pop-run-3.json", 'r') as f:
    pop30_run3 = json.load(f)

# generations = baseline_run1['generations']
pop30_run1_popmax = np.array(pop30_run1['best_fitness'])
pop30_run2_popmax = np.array(pop30_run2['best_fitness'])
pop30_run3_popmax = np.array(pop30_run3['best_fitness'])
pop30_runs_popmaxes = np.vstack((pop30_run1_popmax, pop30_run2_popmax, pop30_run3_popmax))
pop30_means = pop30_runs_popmaxes.mean(axis=0)
pop30_stds = pop30_runs_popmaxes.std(axis=0, ddof=1)

with open("./NEAT_RUN_STATS/silverstone-50pop-run-1.json", 'r') as f:
    pop50_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-50pop-run-2.json", 'r') as f:
    pop50_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-50pop-run-3.json", 'r') as f:
    pop50_run3 = json.load(f)

# generations = baseline_run1['generations']
pop50_run1_popmax = np.array(pop50_run1['best_fitness'])
pop50_run2_popmax = np.array(pop50_run2['best_fitness'])
pop50_run3_popmax = np.array(pop50_run3['best_fitness'])
pop50_runs_popmaxes = np.vstack((pop50_run1_popmax, pop50_run2_popmax, pop50_run3_popmax))
pop50_means = pop50_runs_popmaxes.mean(axis=0)
pop50_stds = pop50_runs_popmaxes.std(axis=0, ddof=1)

with open("./NEAT_RUN_STATS/silverstone-ts5-run-1.json", 'r') as f:
    ts5_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-ts5-run-2.json", 'r') as f:
    ts5_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-ts5-run-3.json", 'r') as f:
    ts5_run3 = json.load(f)

# generations = baseline_run1['generations']
ts5_run1_popmax = np.array(ts5_run1['best_fitness'])
ts5_run2_popmax = np.array(ts5_run2['best_fitness'])
ts5_run3_popmax = np.array(ts5_run3['best_fitness'])
ts5_runs_popmaxes = np.vstack((ts5_run1_popmax, ts5_run2_popmax, ts5_run3_popmax))
ts5_means = ts5_runs_popmaxes.mean(axis=0)
ts5_stds = ts5_runs_popmaxes.std(axis=0, ddof=1)

with open("./NEAT_RUN_STATS/silverstone-ts7-run-1.json", 'r') as f:
    ts7_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-ts7-run-2.json", 'r') as f:
    ts7_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-ts7-run-3.json", 'r') as f:
    ts7_run3 = json.load(f)

# generations = baseline_run1['generations']
ts7_run1_popmax = np.array(ts7_run1['best_fitness'])
ts7_run2_popmax = np.array(ts7_run2['best_fitness'])
ts7_run3_popmax = np.array(ts7_run3['best_fitness'])
ts7_runs_popmaxes = np.vstack((ts7_run1_popmax, ts7_run2_popmax, ts7_run3_popmax))
ts7_means = ts7_runs_popmaxes.mean(axis=0)
ts7_stds = ts7_runs_popmaxes.std(axis=0, ddof=1)

with open("./NEAT_RUN_STATS/silverstone-addnode6-run-1.json", 'r') as f:
    addnode6_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-addnode6-run-2.json", 'r') as f:
    addnode6_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-addnode6-run-3.json", 'r') as f:
    addnode6_run3 = json.load(f)

# generations = baseline_run1['generations']
addnode6_run1_popmax = np.array(addnode6_run1['best_fitness'])
addnode6_run2_popmax = np.array(addnode6_run2['best_fitness'])
addnode6_run3_popmax = np.array(addnode6_run3['best_fitness'])
addnode6_runs_popmaxes = np.vstack((addnode6_run1_popmax, addnode6_run2_popmax, addnode6_run3_popmax))
addnode6_means = addnode6_runs_popmaxes.mean(axis=0)
addnode6_stds = addnode6_runs_popmaxes.std(axis=0, ddof=1)

with open("./NEAT_RUN_STATS/silverstone-addnode9-run-1.json", 'r') as f:
    addnode9_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-addnode9-run-2.json", 'r') as f:
    addnode9_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-addnode9-run-3.json", 'r') as f:
    addnode9_run3 = json.load(f)

# generations = baseline_run1['generations']
addnode9_run1_popmax = np.array(addnode9_run1['best_fitness'])
addnode9_run2_popmax = np.array(addnode9_run2['best_fitness'])
addnode9_run3_popmax = np.array(addnode9_run3['best_fitness'])
addnode9_runs_popmaxes = np.vstack((addnode9_run1_popmax, addnode9_run2_popmax, addnode9_run3_popmax))
addnode9_means = addnode9_runs_popmaxes.mean(axis=0)
addnode9_stds = addnode9_runs_popmaxes.std(axis=0, ddof=1)

with open("./NEAT_RUN_STATS/silverstone-wm40-run-1.json", 'r') as f:
    wm40_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-wm40-run-2.json", 'r') as f:
    wm40_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-wm40-run-3.json", 'r') as f:
    wm40_run3 = json.load(f)

# generations = baseline_run1['generations']
wm40_run1_popmax = np.array(wm40_run1['best_fitness'])
wm40_run2_popmax = np.array(wm40_run2['best_fitness'])
wm40_run3_popmax = np.array(wm40_run3['best_fitness'])
wm40_runs_popmaxes = np.vstack((wm40_run1_popmax, wm40_run2_popmax, wm40_run3_popmax))
wm40_means = wm40_runs_popmaxes.mean(axis=0)
wm40_stds = wm40_runs_popmaxes.std(axis=0, ddof=1)

with open("./NEAT_RUN_STATS/silverstone-wm60-run-1.json", 'r') as f:
    wm60_run1 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-wm60-run-2.json", 'r') as f:
    wm60_run2 = json.load(f)

with open("./NEAT_RUN_STATS/silverstone-wm60-run-3.json", 'r') as f:
    wm60_run3 = json.load(f)

# generations = baseline_run1['generations']
wm60_run1_popmax = np.array(wm60_run1['best_fitness'])
wm60_run2_popmax = np.array(wm60_run2['best_fitness'])
wm60_run3_popmax = np.array(wm60_run3['best_fitness'])
wm60_runs_popmaxes = np.vstack((wm60_run1_popmax, wm60_run2_popmax, wm60_run3_popmax))
wm60_means = wm60_runs_popmaxes.mean(axis=0)
wm60_stds = wm60_runs_popmaxes.std(axis=0, ddof=1)

fig, (ax, ax2, ax3, ax4, ax5) = plt.subplots(5, 1, sharex=True, sharey=True)

# print(baseline_means)
fig.set_size_inches(8, 10.4)

first_hatch = ""
second_hatch = ""
third_hatch = ""
first_color = "black"
second_color = "darkblue"
third_color = "darkred"
first_alpha = 0.5
second_alpha = 0.2
third_alpha = 0.2

baseline_errorbar_1 = ax.errorbar(generations, baseline_means, baseline_stds, linestyle="none", marker='o', color=first_color, elinewidth=1, capsize=1, markersize=2)
species4_errorbar = ax.errorbar(generations, species4_means, species4_stds, linestyle="none", marker='o', color=second_color, elinewidth=1, capsize=1, markersize=2)
species8_errorbar = ax.errorbar(generations, species8_means, species8_stds, linestyle="none", marker='o', color=third_color, elinewidth=1, capsize=1, markersize=2)
species4_errorbar[-1][0].set_linestyle('--')
species8_errorbar[-1][0].set_linestyle('--')
ax.fill_between(generations, baseline_means + baseline_stds, baseline_means - baseline_stds, color=first_color, alpha=first_alpha, hatch=first_hatch, label="species target 6 (BL)")
ax.fill_between(generations, species4_means + species4_stds, species4_means - species4_stds, color=second_color, alpha=second_alpha, hatch=second_hatch, label="Species target 4")
ax.fill_between(generations, species8_means + species8_stds, species8_means - species8_stds, color=third_color, alpha=third_alpha, hatch=third_hatch, label="Species target 8")
ax.set_xlabel("Generation")
ax.set_ylabel("Pop. Best Fitness")
ax.legend()

baseline_errorbar_2 = ax2.errorbar(generations, baseline_means, baseline_stds, linestyle="none", marker='o', color=first_color, elinewidth=1, capsize=1, markersize=2)
pop30_errorbar = ax2.errorbar(generations, pop30_means, pop30_stds, linestyle="none", marker='o', color=second_color, elinewidth=1, capsize=1, markersize=2)
pop50_errorbar = ax2.errorbar(generations, pop50_means, pop50_stds, linestyle="none", marker='o', color=third_color, elinewidth=1, capsize=1, markersize=2)
pop30_errorbar[-1][0].set_linestyle('--')
pop50_errorbar[-1][0].set_linestyle('--')
ax2.fill_between(generations, baseline_means + baseline_stds, baseline_means - baseline_stds, color=first_color, hatch=first_hatch, alpha=first_alpha, label="Population size 40 (BL)")
ax2.fill_between(generations, pop30_means + pop30_stds, pop30_means - pop30_stds, color=second_color, alpha=second_alpha, hatch=second_hatch, label="Population size 30")
ax2.fill_between(generations, pop50_means + pop50_stds, pop50_means - pop50_stds, color=third_color, alpha=third_alpha, hatch=third_hatch, label="Population size 50")
ax2.set_xlabel("Generation")
ax2.set_ylabel("Pop. Best Fitness")
ax2.legend()

baseline_errorbar_3 = ax3.errorbar(generations, baseline_means, baseline_stds, linestyle="none", marker='o', color=first_color, elinewidth=1, capsize=1, markersize=2)
ts5_errorbar = ax3.errorbar(generations, ts5_means, ts5_stds, linestyle="none", marker='o', color=second_color, elinewidth=1, capsize=1, markersize=2)
ts7_errorbar = ax3.errorbar(generations, ts7_means, ts7_stds, linestyle="none", marker='o', color=third_color, elinewidth=1, capsize=1, markersize=2)
ts5_errorbar[-1][0].set_linestyle('--')
ts7_errorbar[-1][0].set_linestyle('--')
ax3.fill_between(generations, baseline_means + baseline_stds, baseline_means - baseline_stds, color=first_color, hatch=first_hatch, alpha=first_alpha, label="Tournament size 3 (BL)")
ax3.fill_between(generations, ts5_means + ts5_stds, ts5_means - ts5_stds, color=second_color, alpha=second_alpha, hatch=second_hatch, label="Tournament size 5")
ax3.fill_between(generations, ts7_means + ts7_stds, ts7_means - ts7_stds, color=third_color, alpha=third_alpha, hatch=third_hatch, label="Tournament size 7")
ax3.set_xlabel("Generation")
ax3.set_ylabel("Pop. Best Fitness")
ax3.legend()

baseline_errorbar_4 = ax4.errorbar(generations, baseline_means, baseline_stds, linestyle="none", marker='o', color=first_color, elinewidth=1, capsize=1, markersize=2)
addnode6_errorbar = ax4.errorbar(generations, addnode6_means, addnode6_stds, linestyle="none", marker='o', color=second_color, elinewidth=1, capsize=1, markersize=2)
addnode9_errorbar = ax4.errorbar(generations, addnode9_means, addnode9_stds, linestyle="none", marker='o', color=third_color, elinewidth=1, capsize=1, markersize=2)
addnode6_errorbar[-1][0].set_linestyle('--')
addnode9_errorbar[-1][0].set_linestyle('--')
ax4.fill_between(generations, baseline_means + baseline_stds, baseline_means - baseline_stds, color=first_color, hatch=first_hatch, alpha=first_alpha, label="Node addition 3% (BL)")
ax4.fill_between(generations, addnode6_means + addnode6_stds, addnode6_means - addnode6_stds, color=second_color, alpha=second_alpha, hatch=second_hatch, label="Node addition 6%")
ax4.fill_between(generations, addnode9_means + addnode9_stds, addnode9_means - addnode9_stds, color=third_color, alpha=third_alpha, hatch=third_hatch, label="Node addition 9%")
ax4.set_xlabel("Generation")
ax4.set_ylabel("Pop. Best Fitness")
ax4.legend()

baseline_errorbar_5 = ax5.errorbar(generations, baseline_means, baseline_stds, linestyle="none", marker='o', color=first_color, elinewidth=1, capsize=1, markersize=2)
wm40_errorbar = ax5.errorbar(generations, wm40_means, wm40_stds, linestyle="none", marker='o', color=second_color, elinewidth=1, capsize=1, markersize=2)
wm60_errorbar = ax5.errorbar(generations, wm60_means, wm60_stds, linestyle="none", marker='o', color=third_color, elinewidth=1, capsize=1, markersize=2)
wm40_errorbar[-1][0].set_linestyle('--')
wm60_errorbar[-1][0].set_linestyle('--')
ax5.fill_between(generations, baseline_means + baseline_stds, baseline_means - baseline_stds, color=first_color, hatch=first_hatch, alpha=first_alpha, label="Weight mutation 80% (BL)")
ax5.fill_between(generations, wm40_means + wm40_stds, wm40_means - wm40_stds, color=second_color, alpha=second_alpha, hatch=second_hatch, label="Weight mutation 40%")
ax5.fill_between(generations, wm60_means + wm60_stds, wm60_means - wm60_stds, color=third_color, alpha=third_alpha, hatch=third_hatch, label="Weight mutation 60%")
ax5.set_xlabel("Generation")
ax5.set_ylabel("Pop. Best Fitness")
ax5.legend()

# fig.text(0.04, 0.5, 'common Y', va='center', rotation='vertical')
# plt.ylabel("Population Best Fitness")

fig.tight_layout()
plt.subplots_adjust(wspace=0)
plt.subplots_adjust(hspace=.0)
fig.savefig("stacked_sensitivity_plots.png")
plt.show()


from django.shortcuts import render, get_object_or_404, redirect
from django.http import HttpResponseForbidden, HttpResponse
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.contrib import messages
from django.utils import timezone
from .models import Post, Comment
from .forms import BlogPostForm, CommentForm

def get_posts(request):
	"""
	Create a view that will return a list
	of posts that were published prior to 'now'
	and render them to the 'blogposts.html
	"""
	posts = Post.objects.filter(published_date__lte=timezone.now()
		).order_by('-published_date')
	return render(request, "blogposts.html", {'posts': posts})

# def post_detail(request, pk):
# 	"""
# 	Create a viw that returns a single
# 	Post object based on the post ID (pk) and
# 	render it to the 'postdetail.html' template.
# 	Or return a 404 error if the post is not found.
# 	"""
# 	post = get_object_or_404(Post, pk=pk)
# 	post.views += 1
# 	post.save()
# 	comments = Comment.objects.filter(active=True).order_by('-created_on')
# 	comment_form = CommentForm()
# 	return render(request, "postdetail.html", {'post': post,
# 												'comments': comments,
# 												'comment_form': comment_form})

def post_detail(request, pk):
	template_name = "postdetail.html"
	post = get_object_or_404(Post, pk=pk)
	post.views += 1
	post.save()
	comments = Comment.objects.filter(active=True).order_by('-created_date')
	comment = None
	# Processing post requests
	if request.method == 'POST':
		comment_form = CommentForm(request.POST)
		if comment_form.is_valid():
			comment = comment_form.save(commit=False)
			comment.post = post
			comment.owner = request.user
			comment.save()
			return redirect('post_detail', pk=post.pk)
	else:
		comment_form = CommentForm()

	return render(request, template_name, {'comment_form': comment_form,
											'comment': comment,
											'comments': comments, 'post': post})

# def create_or_edit_post(request, pk=None):
#     """
#     Create a view that allows us to create
#     or edit a post depending if the Post ID
#     is null or not
#     """
#     post = get_object_or_404(Post, pk=pk) if pk else None
#     if request.method == "POST":
#         form = BlogPostForm(request.POST, request.FILES, instance=post)
#         if form.is_valid():
#             post = form.save()
#             return redirect(post_detail, post.pk)
#     else:
#         form = BlogPostForm(instance=post)
#     return render(request, 'blogpostform.html', {'form': form})


import copy
import collections

__all__ = ['Parameter', 'ParameterDict', 'OrderedParameterDict',
           'BaseParameterDict', 'SyncedParameters']

class Parameter:
    """A named parameter with a optional value.

    The value can be changed at any time and very basic
    operations can be queued and evaluated on demand.

    Parameters
    ----------
    name : str
        Name of parameter.
    value : int or float, optional, default=None
        Value of parameter. Other types may work but not supported.

    Attributes
    ----------
    name : str
        Name of parameter.
    value : int or float
        Value of parameter, without operations applied.
    operations : list
        List of operations to apply.

    Raises
    ------
    ValueError
        The parameter is evaluated without a value.
    NotImplementedError
        Operations which require parentheses are applied.

    See Also
    --------
    Parameters : Collection of synced ``Parameter`` objects.

    Notes
    -----
    Basic operations can be applied to this object, i.e., ``+``, ``-``, ``*`` and ``/``.
    The ``Parameter`` object must be the left-most entity in an expression.
    Expressions which require parentheses are not supported.
    The only types that are supported in expressions are ``Parameter``, ``float``
    and ``int``.
    Any type should work as a value for ``Parameter`` (such as an array), however,
    only ``float`` and ``int`` are supported.
    See the examples for more details in how it works.

    Examples
    --------
    >>> Parameter('x')
    'x'
    >>> Parameter('x', 12)
    '12'
    >>> Parameter('x') + 1
    'x+1'
    >>> Parameter('x', 1) + 1
    'x+1'
    >>> a = Parameter('y') * 2
    >>> a
    'y*2'
    >>> a.value = 5
    >>> a == 10
    True
    >>> a.eval()
    10
    >>> (a * 10).eval()
    100
    >>> a * 10 + 1.5
    'y*2*10+1.5'
    >>> (a + 10) * 2
    Traceback (most recent call last):
     ...
    NotImplementedError: Parameter equations with brackets are not supported.
    """
    def __init__(self, name, value=None):
        self.name = str(name)
        self.value = value
        self.operations = []

    def __add__(self, other):
        return self.apply_operation('+', other)

    def __sub__(self, other):
        return self.apply_operation('-', other)

    def __mul__(self, other):
        return self.apply_operation('*', other)

    def __truediv__(self, other):
        return self.apply_operation('/', other)

    def apply_operation(self, op, other):
        """Apply an operation to the ``Parameter``.

        Parameters
        ----------
        op : str
            Operation symbol to apply. ``+``, ``-``, ``*`` or ``/``.
        other : float or int
            Number on RHS.

        Returns
        -------
        obj
            A copy of ``self`` with the operation added to the queue.

        Notes
        -----
        You should only use this method directly if a builtin Python operation
        is not otherwise implemented in this class, i.e., do ``a + 1`` and not
        ``a.apply_operation('+', 1)``.
        """
        obj = self.copy()  # Operate on a copy
        obj.operations += [(op, other)]  # Add to queue
        obj.verify()  # Verify no parentheses
        return obj

    def __eq__(self, other):
        # Test equality against the expression's numeric value.
        # If the parameter doesn't have a value,
        # test equality against the string representation
        # of the unevaluated expression.
        try:
            exp = self.eval()
        except ValueError:
            exp = str(self)
        return exp == other

    @property
    def value_or_name(self):
        """Returns its value if set, otherwise returns its name."""
        if self.value is None:
            return self.name
        else:
            return self.value

    def eval(self):
        """Evaluate the expression using the parameter's current value."""
        if self.value is None:
            raise ValueError('Cannot evaluate without a value.')
        return eval(
            str(self),
            {'__builtins__': None},
            {self.name: self.value}
        )

    def __str__(self):
        if len(self.operations) == 0:
            return str(self.value_or_name)
        ret = str(self.name)
        for op, val in self.operations:
            if isinstance(val, Parameter):
                val = val.eval()
            ret += f'{op}{val}'
        return ret

    def __repr__(self):
        return f"'{str(self)}'"

    def verify(self):
        """Enforce order of operations."""
        disable_mul_div = False
        for op, _ in self.operations:
            if op in ('+', '-'):
                disable_mul_div = True
            elif op in ('*', '/') and disable_mul_div:
                raise NotImplementedError(
                    'Parameter equations with brackets are not supported.'
                )

    def copy(self):
        """Return a copy of the parameter."""
        return self.__copy__()

    def __copy__(self):
        obj = self.__class__(copy.copy(self.name), copy.copy(self.value))
        obj.operations = copy.deepcopy(self.operations)
        return obj

    def __deepcopy__(self, memodict={}):
        return self.__copy__()

class SyncedParameters:
    """Database for keeping :class:`Parameter` objects in sync.

    :class:`Parameter` objects with the same name can be linked with this class
    and kept in sync with each other.

    Examples
    --------
    >>> a = Parameter('x') + 1
    >>> b = Parameter('x', 2) * 3
    >>> c = Parameter('y')

    Now that :class:`Parameter` objects have been created,
    we can keep them in sync.

    >>> s = SyncedParameters()
    >>> s.track_object(a)
    >>> s.track_object(b)
    >>> s.track_object(c)

    Notice how the value of `a` has been synced to match the
    value provided in `b`.

    >>> a == 3
    True

    The value of a named parameter can be updated and the
    change is propagated to all :class:`Parameter` objects
    of that name.

    >>> s.update_parameter('x', 3)
    >>> a == 4
    True
    >>> b == 9
    True
    """
    @property
    def _tracked(self):
        """Dictionary of tracked :class:`Parameters` grouped by name.

        Notes
        -----
        Has the form: ``{Parameter.name: {'value': Parameter.value, 'objects': List[Parameter]}}``.
        All ``Parameter.value`` of the same ``Parameter.name`` are
        identical in both ``'value'`` and all in ``'objects'``.
        """
        if not hasattr(self, '_tracked_objs'):
            self._tracked_objs = {}
        return self._tracked_objs

    def track_object(self, obj):
        """Add a new :class:`Parameter` object to keep in sync.

        Parameters
        ----------
        obj : :class:`Parameter`
            The parameter object to keep in sync.

        Notes
        -----
        Any objects added must not have a value that contradicts the existing
        value of the parameter (if set). Objects added which have no value set
        will inherit the value of the other parameters of the same name.
        """
        if self.exists(obj.name):  # If parameter already registered...
            if obj.value is not None:  # ...copy incoming value to existing.
                # (No more than one value should be associated with each name.)
                if self.has_value(obj.name) and obj.value is not self.get_parameter(obj.name):
                    raise ValueError(f"'{obj.name}' value {obj.value} does not match "
                                     f"existing value {self.get_parameter(obj.name)}.")
                self.update_parameter(obj.name, obj.value)
            elif self.has_value(obj.name):  # ...copy existing value to incoming.
                obj.value = self.get_parameter(obj.name)
            self._tracked[obj.name]['objects'] += [obj]
        else:
            self._tracked[obj.name] = {'value': obj.value, 'objects': [obj]}

    def update_parameter(self, name, value):
        """Update the value of a named parameter.

        All tracked :class:`Parameter` objects of this name will be updated.

        Parameters
        ----------
        name : string
            Name of parameter to update.
        value : float or int
            New value of parameter. Other types may work.
        """
        self._tracked[name]['value'] = value
        for obj in self._tracked[name]['objects']:
            obj.value = value

    def get_parameter(self, name):
        """Get the value of the named parameter.

        Parameters
        ----------
        name : str
            Name of parameter to get value of.

        Returns
        -------
        value
            The current value of the parameter. ``None`` if not set.
        """
        return self._tracked[name]['value']

    def has_value(self, name: str) -> bool:
        """Whether the named parameter has a value set."""
        return False if self.get_parameter(name) is None else True

    def exists(self, name: str) -> bool:
        """Whether any parameters of a particular name are tracked."""
        try:
            self._tracked[name]
        except KeyError:
            return False
        else:
            return True

class BaseParameterDict(SyncedParameters):
    """
    A base class for dictionaries of :class:`Parameter` objects.

    The same parameters existing across multiple dictionary
    values can be kept in sync with each other.
    For a :class:`Parameter` object to be kept in sync it must be
    located in the dictionary such that ``{key: Parameter}`` or
    ``{key: List[Parameter]}``.
    """
    def __setitem__(self, key, value):
        if isinstance(value, Parameter):
            self.track_object(value)
        elif isinstance(value, list):
            for item in value:
                if isinstance(item, Parameter):
                    self.track_object(item)
        super().__setitem__(key, value)

    def eval(self):
        """Return a copy of the dictionary with all :class:`Parameter` objects evaluated."""
        edict = self.__class__.__bases__[-1]()
        for key, value in self.items():
            if isinstance(value, Parameter):
                value = value.eval()
            elif isinstance(value, list):
                lst = []
                for i, v in enumerate(value):
                    if isinstance(v, Parameter):
                        lst += [v.eval()]
                    else:
                        lst += [v]
                value = lst
            edict[key] = value
        return edict

class ParameterDict(BaseParameterDict, collections.UserDict):
    """
    An unordered dictionary of :class:`Parameter` objects.

    The same parameters existing across multiple dictionary
    values can be kept in sync with each other.
    For a :class:`Parameter` object to be kept in sync it must be
    located in the dictionary such that ``{key: Parameter}`` or
    ``{key: List[Parameter]}``.

    Examples
    --------
    >>> d = ParameterDict({
    ...     'a': Parameter('x') + 1,
    ...     'b': [2, Parameter('x'), 5],
    ...     'c': {1, 2, 3},
    ... })
    >>> d
    {'a': 'x+1', 'b': [2, 'x', 5], 'c': {1, 2, 3}}
    >>> d.update_parameter('x', 1)
    >>> d.eval()
    {'a': 2, 'b': [2, 1, 5], 'c': {1, 2, 3}}
    """

class OrderedParameterDict(BaseParameterDict, collections.OrderedDict):
    """
    An ordered dictionary of :class:`Parameter` objects.

    The same parameters existing across multiple dictionary
    values can be kept in sync with each other.
    For a :class:`Parameter` object to be kept in sync it must be
    located in the dictionary such that ``{key: Parameter}`` or
    ``{key: List[Parameter]}``.

    Examples
    --------
    >>> d = OrderedParameterDict([
    ...     ('a', Parameter('x') + 1),
    ...     ('b', [2, Parameter('x'), 5]),
    ...     ('c', {1, 2, 3}),
    ... ])
    >>> d
    OrderedParameterDict([('a', 'x+1'), ('b', [2, 'x', 5]), ('c', {1, 2, 3})])
    >>> d.update_parameter('x', 1)
    >>> d.eval()
    OrderedDict([('a', 2), ('b', [2, 1, 5]), ('c', {1, 2, 3})])
    """


import requests
from bs4 import BeautifulSoup
from bs4.element import Comment
import re
import validators
import time
from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from TopicExtraction import predictCategory

#Inspired by https://stackoverflow.com/questions/1936466/beautifulsoup-grab-visible-webpage-text
def tag_visible(element):
    if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
        return False
    if isinstance(element, Comment):
        return False
    return True

def stringExtract(soup):
    soup = BeautifulSoup(soup, 'html.parser')
    texts = soup.findAll(text=True)
    visibleText = filter(tag_visible, texts)
    allText = []

    for t in visibleText:
        t = str(t.strip())
        t = re.split(r'([(|\]])+(|\. )+' ,t)

        for text in t:
            text = re.sub(r'[\)|\]]', '',text)
            if len(text)>0:
                allText.append(text)
    return allText

#Iteratively depth checks surrounding elements to a particular element (Button)
def depthCheckHTML(soup,word,buttonClassIds):
    if not isinstance(soup,str):
        soup = str(soup)
    if not isinstance(word,str):
        word = str(word)

    checkLevel = 0
    newRowCount = 0
    stoppingCond = False
    soupLength = len(soup)

    while not stoppingCond :
        #print("***CHECK LEVEL ",checkLevel, "***")

        if checkLevel == 0:
            startIndex = soup.find(word)
            endIndex = startIndex+len(word)

        rowCount = newRowCount
        deepSoup = soup[startIndex:endIndex+1]

        interiorTagCount = 0
        foundStart = False
        foundEnd = False

        while foundStart == False:
            startIndex = startIndex -1
            if startIndex<1:
                return soup,deepSoup
            if soup[startIndex] == "<":
                if soup[startIndex+1] == "/":
                    interiorTagCount+=2
                if interiorTagCount ==0:
                    foundStart = True
                interiorTagCount -=1
            elif soup[startIndex] == ">":
                if soup[startIndex-1] == "/":
                    interiorTagCount += 1

        interiorTagCount = 0

        while foundEnd == False:
            endIndex = endIndex +1
            if endIndex >= soupLength:
                return soup, deepSoup
            if soup[endIndex] == ">":
                if interiorTagCount ==0:
                    foundEnd = True
                interiorTagCount-=1
            elif soup[endIndex] == "<":
                if soup[endIndex+1] != "/":
                    interiorTagCount += 1

        eventHTML = BeautifulSoup(soup[startIndex:endIndex+1], 'html.parser')
        buttonsInSoup = eventHTML.find_all('button')

        newRowCount = eventHTML.prettify().count('\n')+1
        #print("newRow = ",newRowCount," oldRow = ",rowCount)

        #If the number of rows is increasing by more than 10 this turn and it isn't the first turn then stop the depth search
        if (newRowCount-rowCount)>10 and rowCount!=0:
            stoppingCond = True
            #print("STOP DEPTH-SEARCH: Row Increase Too Large")
            return soup,deepSoup

        #If there are more than one button then compare the button classes with those of our buttonClassIds to see if we are including
        #repeats of the same button.
        elif len(buttonsInSoup)!=1:
            buttonsInSoup.pop() #The first element is always the target button, so this is removed for this check

            for buttons in buttonsInSoup:
                classInSoup = " ".join(buttons['class'])
                for classInList in buttonClassIds:
                    if classInSoup == classInList:
                        stoppingCond = True
                        #print("STOP DEPTH-SEARCH: Too Many Buttons")
                        return soup,deepSoup

        checkLevel+=1
    return soup,deepSoup

#Returns the number of events and prints these events out topic by topic
def extractEvent(baseUrl,URL,soup):
    soup = str(soup)
    soup = BeautifulSoup(soup, 'html.parser')

    buttonMentions = soup.find_all('button',{"class":True})
    buttonClassIds = []
    buttonCount=0
    eventCount=0
    topicCount=0

    for button in buttonMentions:
        buttonCount+=1
        #print("\nBUTTON MENTION ",buttonCount)
        buttonClassIds.append(" ".join(button['class']))

        depthCheckResults = depthCheckHTML(soup,button,buttonClassIds)
        soup = depthCheckResults[0]
        deepSoup = depthCheckResults[1]

        soupStrExtract = stringExtract(soup)
        deepSoupStrExtract = stringExtract(deepSoup)

        if len(soupStrExtract) >0 and len(deepSoupStrExtract)>0:
            topics = predictCategory(deepSoupStrExtract,soupStrExtract)
            if topics>2:
                eventCount+=1
                topicCount +=topics

    return eventCount , topicCount


# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Main model file for SegFormer. """

import torch
from omegaconf import OmegaConf

class SFModel(object):  # pylint: disable=too-many-ancestors
    """Pytorch Module for SegFormer Model."""

    def __init__(self, experiment_spec, phase=None, num_classes=None):
        """Init Segformer Model module.
        Args:
            experiment_spec (Dict): Dictionary of the spec parameters.
            phase (str): Indicates train, val or test phase.
            num_classes (int): Number of classes.
        """
        super().__init__()

        self.experiment_spec = experiment_spec
        self.dataset_config = experiment_spec["dataset"]
        self.sf_config = experiment_spec["model"]
        self.phase = phase
        if phase == "train":
            self.sf_optim = experiment_spec["train"]["trainer"]["sf_optim"]
            self.sf_optim_cfg = OmegaConf.to_container(self.sf_optim)
            self.lr_config = OmegaConf.to_container(experiment_spec["train"]["trainer"]["lr_config"])
            self.validation_interval = experiment_spec["train"]["validation_interval"]
            self.find_unused_parameters = experiment_spec["train"]["trainer"]["find_unused_parameters"]

        self.train_cfg = None
        self.test_cfg = OmegaConf.to_container(self.sf_config["test_cfg"])
        self.model_cfg = OmegaConf.to_container(self.sf_config)
        self.model_cfg["pretrained"] = self.model_cfg["pretrained_model_path"]
        self.model_cfg.pop("pretrained_model_path")
        self.model_cfg["type"] = "EncoderDecoder"
        self.model_cfg["decode_head"]["type"] = "SegFormerHead"
        self.model_cfg["backbone"]["style"] = "pytorch"
        self.model_cfg.pop("input_height")
        self.model_cfg.pop("input_width")
        self.backbone = self.model_cfg["backbone"]["type"]

        self.channels_map = {"mit_b0": [32, 64, 160, 256],
                             "fan_tiny_8_p4_hybrid": [128, 256, 192, 192],
                             "fan_large_16_p4_hybrid": [128, 256, 480, 480],
                             "fan_small_12_p4_hybrid": [128, 256, 384, 384],
                             "fan_base_16_p4_hybrid": [128, 256, 448, 448], }
        if self.backbone in self.channels_map:
            self.model_cfg["decode_head"]["in_channels"] = self.channels_map[self.backbone]
        if "fan" in self.backbone:
            self.model_cfg["decode_head"]["channels"] = 256
        self.export = False
        self.model_cfg["backbone"]["export"] = self.export
        self.test_cfg["export"] = self.export
        self.model_cfg["train_cfg"] = self.train_cfg
        if self.test_cfg["mode"] == "whole":
            self.test_cfg.pop("crop_size")
            self.test_cfg.pop("stride")
        self.model_cfg["test_cfg"] = self.test_cfg
        self.model_cfg["decode_head"]["export"] = self.export
        self.model_cfg["decode_head"]["num_classes"] = num_classes
        self.num_classes = num_classes
        self.distributed = experiment_spec["train"]["exp_config"]["distributed"]
        self.checkpoint_loaded = None
        self.tmp_ckpt = None
        self.max_iters = 1
        self.resume_ckpt = None
        self.checkpoint_interval = 1000

    def _convert_batchnorm(self, module):
        """ Convert Sync BN during Export."""
        module_output = module
        if isinstance(module, torch.nn.SyncBatchNorm):
            module_output = torch.nn.BatchNorm2d(module.num_features, module.eps,
                                                 module.momentum, module.affine,
                                                 module.track_running_stats)
            if module.affine:
                module_output.weight.data = module.weight.data.clone().detach()
                module_output.bias.data = module.bias.data.clone().detach()
                # keep requires_grad unchanged
                module_output.weight.requires_grad = module.weight.requires_grad
                module_output.bias.requires_grad = module.bias.requires_grad
            module_output.running_mean = module.running_mean
            module_output.running_var = module.running_var
            module_output.num_batches_tracked = module.num_batches_tracked
        for name, child in module.named_children():
            module_output.add_module(name, self._convert_batchnorm(child))
        del module
        return module_output