'''
INPUT:
The loop_tgz function below looks in the specified environment directory for all available files with a filename in
the format <game_type>.MMMMYY.tgz. This is a compressed "zip" file type that the loop_tgz function extracts into
uncompressed, readable files.

Each .tgz file extracts a "file group" of the following structure:
./<game_type>/MMMMYYY/hdb
./<game_type>/MMMMYYY/hroster
./<game_type>/MMMMYYY/pdb/pdb.player1 <- where "player1" is one player's username
./<game_type>/MMMMYYY/pdb/pdb.player2
./<game_type>/MMMMYYY/pdb/pdb.player3
...
./<game_type>/MMMMYYY/pdb/pdb.playerN

For the hand below, the various files have records that look like the following:
hdb file:

HAND INFORMATION (in hdb file)
------------------------------
timestamp      hand #     #players/starting potsize
         dealer    #play flop    turn    river  showdn     board
820830094  20  1163  2  2/20    2/40    2/80     2/80     Qc 4s 6s 5d 4d

ROSTER INFORMATION (in hroster file)
------------------------------------
820830094  2 Jak num

PLAYER INFORMATION (in pdb.* files)
-----------------------------------
player             #play prflop    turn         bankroll    winnings
          timestamp    pos   flop       river           action  cards
Jak       820830094  2  1 Bc  kc    kc    k          850   40   80 7c Ac
num       820830094  2  2 Bk  b     b     k         1420   40    0 9h Kh

In English, the database entry says that 2 players are at the table: "Jak" and "num".
Pre-flop, Jak makes a small blind (B) $5, num makes a big blind (B) $10, Jak calls (c), and
num checks (k).  The pot is $20 and we have 2 players.
The flop comes Qc 4s 6s, Jak checks (k), num bets (b) $10, Jak calls (c). The pot is now $40.
On the turn, 5d, Jak checks (k), num bets (b) $20, and Jak calls (c). The pot is now $80
On the river, 4d, Jak checks (k), and num checks (k). The pot remains $80.
At the showdown, Jak wins the hand with a pair of board 4's w/ Ace high versus num's board 4's w/ King high.
As a result, Jak wins the $80 pot.
Note that we don't know the cards of any players who fold before the showdown.
Also note that the format does not record exact bet amounts in pot-limit or no-limit, though
they can often be inferred from the total pot size after each round.

PROCESSING:
All valid hands found in the processed .tgz files are parsed into a list of "hands" dictionaries by running the
loop_file_groups function, which runs all of the following functions on each file group, one at a time:

- parse_hdb_file: parses the hdb file looking for a hand identifier, betting rounds, and board cards
- parse_hroster_file: parses the hroster file looking for each of the players that played each hand
- loop_pdb_files: loops through all of the pdb files in the pdb folder, identifying each one by the username on the end
    - parse_pdb_files: parses an individual pdb file, to determine what that player did with each hand she played
- append_hands_list_to_json_file: takes a list of all of the valid hands found in the file group, "dumps" them from
    Python dictionary format to JSON, and appends them to an output file, "cprg_hands.json"

OUTPUT:

A full element of the Python dictionary list that this code produces looks like this (from the example inputs above):

{'_id': 'holdem_199601_820830094',
 'board': ['Qc', '4s', '6s', '5d', '4d'],
 'dealer': 20,
 'game': 'holdem',
 'hand_num': 1163,
 'num_players': 2,
 'players': {'Jak': {'action': 40,
                     'bankroll': 850,
                     'bets': [{'actions': ['B', 'c'],
                                      'stage': 'p'},
                                     {'actions': ['k', 'c'],
                                      'stage': 'f'},
                                     {'actions': ['k', 'c'],
                                      'stage': 't'},
                                     {'actions': ['k'],
                                      'stage': 'r'}],
                     'pocket_cards': ['7c', 'Ac'],
                     'pos': 1,
                     'user': 'Jak',
                     'winnings': 80},
             'num': {'action': 40,
                     'bankroll': 1420,
                     'bets': [{'actions': ['B', 'k'],
                                      'stage': 'p'},
                                     {'actions': ['b'], 'stage': 'f'},
                                     {'actions': ['b'], 'stage': 't'},
                                     {'actions': ['k'],
                                      'stage': 'r'}],
                     'pocket_cards': ['9h', 'Kh'],
                     'pos': 2,
                     'user': 'num',
                     'winnings': 0}},
 'pots': [{'num_players': 2, 'stage': 'f', 'size': 20},
          {'num_players': 2, 'stage': 't', 'size': 40},
          {'num_players': 2, 'stage': 'r', 'size': 80},
          {'num_players': 2, 'stage': 's', 'size': 80}]}

The JSON file that gets produced has one of the above "hand" dictionaries per line, "dumped" into JSON format,
and ready for loading into MongoDB.
'''

# Parses "hdb" file from the IRC Poker Database http://poker.cs.ualberta.ca/irc_poker_database.html
from builtins import str
from ColorPrint import print
import os
from tarfile import TarFile
import re
import codecs
import json

# ENVIRONMENT VARIABLES -- CHANGE THESE TO FIT YOUR ENVIRONMENT
tgz_extract_directory = "/Users/allenfrostline/Downloads/"
OUTFILE = tgz_extract_directory + "hands.json"
LOCAL_OS = "mac"  # valid values are "mac" or "pc"
# END ENVIRONMENT VARIABLES

# Code to use the local_os variable to control logic
if LOCAL_OS == "mac":
    SLASH = "/"
else:
    SLASH = "\\"

# Global variables
pot_cats = ["f", "t", "r", "s"]  # f=flop, t=turn, r=river, s=showdown
deck = {
    'A': 'ace',
    'K': 'king',
    'Q': 'queen',
    'J': 'jack',
    'T': '10'
}  # No longer in use; here as a reference
suits = {
    'c': 'clubs',
    's': 'spades',
    'h': 'hearts',
    'd': 'diamonds'
}  # No longer in use; here as a reference
bet_action_codes = {
    '-': 'no action',
    'B': 'blind bet',
    'f': 'fold',
    'k': 'check',
    'b': 'bet',
    'c': 'call',
    'r': 'raise',
    'A': 'all-in',
    'Q': 'quits game',
    'K': 'kicked from game'
}
# bet_action_codes is no longer in use but here as a reference
bet_action_cats = ["p", "f", "t", "r"]  # p=pre-flop, f=flop, t=turn, r=river
folder_search_re = re.compile(r'\d{6}$', re.IGNORECASE)
tgz_search_re = re.compile(r'^\S*.\d{6}.tgz$', re.IGNORECASE)
valid_game_types = {
    "holdem", "holdem1", "holdem2", "holdem3", "holdemii", "holdempot",
    "nolimit", "tourney"
}

def parse_hdb_file(hdb_file, hands, invalid_keys):

    try:
        split_filename = hdb_file.split(SLASH)
        id_prefix = split_filename[-3] + "_" + split_filename[-2] + "_"
        with open(hdb_file, "r") as hdb:
            for line in hdb:
                hand = {}
                pot_data = []
                board = []
                line_parts = line.strip("\n").split(" ")
                line_parts = [elem for elem in line_parts if elem != '']
                _id = id_prefix + line_parts[0]
                hand["_id"] = _id
                hand["game"] = split_filename[-3]
                hand["dealer"] = int(line_parts[1])
                hand["hand_num"] = int(line_parts[2])
                hand["num_players"] = int(line_parts[3])
                for lp in line_parts[4:8]:
                    pot_data.append(lp)
                for card in line_parts[8:]:
                    board.append(card)

                pots = []
                i = 0
                for p in pot_data:
                    pot = {}
                    pot["stage"] = pot_cats[i]
                    if len(p.split("/")) == 2:
                        pot["num_players"] = int(p.split("/")[0])
                        pot["size"] = int(p.split("/")[1])
                    else:
                        invalid_keys.add(_id)
                    pots.append(pot)
                    i = i + 1
                ''' Old Code
                board = []
                for b in board_card_data:
                    board_card = {}
                    if b != "":

                        if board_card_data.index(b) + 1 <= 3:
                            board_card["stage"] = "flop"
                        elif board_card_data.index(b) + 1 == 4:
                            board_card["stage"] = "turn"
                        elif board_card_data.index(b) + 1 == 5:
                            board_card["stage"] = "river"
                        if b[0] in deck.keys():
                            board_card["value"] = deck[b[0]]
                        else:
                            board_card["value"] = b[0]
                        board_card["suit"] = suits[b[1]]
                        board.append(board_card)
                '''

                hand["pots"] = pots
                hand["board"] = board
                hands[_id] = hand
        hdb.close()
        return hands, id_prefix, invalid_keys

    except (KeyError, ValueError):
        invalid_keys.add(_id)
        hdb.close()
        return hands, id_prefix, invalid_keys

def parse_hroster_file(hroster_file, id_prefix, hands, invalid_keys):

    try:
        with open(hroster_file, "r") as hroster:
            for line in hroster:
                players = {}
                line_parts = line.strip("\n").split(" ")
                line_parts = [elem for elem in line_parts if elem != '']
                _id = id_prefix + line_parts[0]
                player_data = line_parts[2:]
                for p in player_data:
                    # If on pc replace "|" with "_" in the player's name
                    if LOCAL_OS == "pc":
                        p = re.sub(r'[|]', '_', p)
                    # end fix 1/4/16
                    player = {}
                    player["user"] = p
                    players[p] = player
                if _id in hands:
                    hands[_id]["players"] = players
                else:
                    invalid_keys.add(_id)
        hroster.close()
        return hands, invalid_keys

    except KeyError:
        invalid_keys.add(_id)
        hroster.close()
        return hands, invalid_keys

def parse_pdb_file(pdb_file, id_prefix, hands, invalid_keys):

    try:
        username = pdb_file.split(".")[-1]
        with open(pdb_file, "r") as pdb:
            for line in pdb:
                line_parts = line.strip("\n").split(" ")
                line_parts = [elem for elem in line_parts if elem != '']

                _id = id_prefix + line_parts[1]
                position = line_parts[3]

                bet_actions = []
                i = 0
                for item in line_parts:
                    bet_action = {}
                    bet_action["actions"] = []
                    if line_parts.index(item) >= 4 and line_parts.index(
                            item) <= 7:
                        for b in item:
                            bet_action["actions"].append(b)
                        bet_action["stage"] = bet_action_cats[i]
                        bet_actions.append(bet_action)
                        i = i + 1
                bankroll, action, winnings = line_parts[8:11]

                player_cards = []
                if len(line_parts) == 13:
                    for card in line_parts[11:13]:
                        player_cards.append(card)
                        ''' Old code
                        if item[0] in deck.keys():
                            player_card["value"] = deck[item[0]]
                        else:
                            player_card["value"] = item[0]
                        player_card["suit"] = suits[item[1]]
                        player_cards.append(player_card)
                        '''

                if _id in hands:
                    if _id not in invalid_keys:
                        if username in hands[_id]["players"]:
                            hands[_id]["players"][username][
                                "bets"] = bet_actions
                            hands[_id]["players"][username]["bankroll"] = int(
                                bankroll)
                            hands[_id]["players"][username]["action"] = int(
                                action)
                            hands[_id]["players"][username]["winnings"] = int(
                                winnings)
                            hands[_id]["players"][username][
                                "pocket_cards"] = player_cards
                            hands[_id]["players"][username]["pos"] = int(
                                position)
                        else:
                            invalid_keys.add(_id)
                else:
                    invalid_keys.add(_id)
        pdb.close()
        return hands, invalid_keys

    except IndexError:
        invalid_keys.add(_id)
        pdb.close()
        return hands, invalid_keys
    except KeyError:
        invalid_keys.add(_id)
        pdb.close()
        return hands, invalid_keys
    except ValueError:
        invalid_keys.add(_id)
        pdb.close()
        return hands, invalid_keys

def loop_pdb_files(pdb_file_directory, hands_col, id_prefix, invalid_keys):
    for root, dirs, files in os.walk(pdb_file_directory, topdown=False):
        for name in files:
            pdb_file = os.path.join(root, name)
            hs = parse_pdb_file(pdb_file, id_prefix, hands_col, invalid_keys)
    return hands_col, invalid_keys

def fix_players_list(hands_list):
    fixed_hands_list = []
    for h in hands_list:
        if "players" in h:
            players_list = list(h["players"].values())
            h["players"] = players_list
            fixed_hands_list.append(h)
    return fixed_hands_list

def loop_file_groups(file_groups):
    total_to_process = len(file_groups)
    i = 1
    running_total = 0
    for fg in file_groups:
        try:
            hands = {}
            hands_list = []
            invalid_keys = set()
            print("Processing " + fg + " (file group #" + str(i) + " of " + str(
                total_to_process) + ")")
            hdb_file = fg + "hdb"
            hroster_file = fg + "hroster"
            pdb_directory = fg + "pdb/"
            hands, idp, inv_keys = parse_hdb_file(hdb_file, hands, invalid_keys)
            hands, inv_keys = parse_hroster_file(hroster_file, idp, hands,
                                                 inv_keys)
            hands, inv_keys = loop_pdb_files(pdb_directory, hands, idp, inv_keys)
            hands = {key: hands[key] for key in hands if key not in inv_keys}
            hands_list = fix_players_list(list(hands.values()))
            append_hands_list_to_json_file(hands_list)
            print(str(len(hands_list)) + " valid hands added to JSON file, " + str(
                len(inv_keys)) + " invalid hands", color='green')
            running_total = running_total + len(hands_list)
            print(str(running_total) + " total hands added so far", color='yellow')
            print("Finished processing " + fg + '\n')
        except IndexError:
            print('Failed to process ' + fg + '\n')
        i = i + 1

def loop_tgz(extract_dir):
    try:
        file_groups = []
        for root, dirs, files in os.walk(tgz_extract_directory, topdown=False):
            for file in files:
                tgz = tgz_search_re.search(file)
                if tgz:
                    tgz_file = os.path.join(root, file)
                    game_type = tgz_file.split(".")[-3].split(SLASH)[-1]
                    file_yearmonth = tgz_file.split(".")[-2]
                    if game_type in valid_game_types:
                        print("Extracting " + tgz_file)
                        tar = TarFile.open(tgz_file)
                        if LOCAL_OS == "pc":
                            for f in tar:
                                # Replace "|" with "_".
                                f.name = re.sub(r'[|]', '_', f.name)
                        tar.extractall(extract_dir)
                        tar.close()
                        file_groups.append(tgz_extract_directory + game_type +
                                           SLASH + file_yearmonth + SLASH)
                    else:
                        print("Skipping " + tgz_file + " because it is for an invalid game type", color='red')
        return file_groups
    except IOError:
        # invalid_keys.add(_id)
        tar.close()
        return file_groups

def append_hands_list_to_json_file(hands_list):
    # db = connect_to_cloud_for_upload()
    with codecs.open(OUTFILE, "a") as fo:
        for hand in hands_list:
            fo.write(json.dumps(hand) + "\n")
    fo.close()

file_groups = loop_tgz(tgz_extract_directory)
print()
if os.path.isfile(OUTFILE): os.remove(OUTFILE)
loop_file_groups(file_groups)
print("Finished.")


#!/usr/bin/env python3

from app import lambda_handler
import os, boto3, argparse

def mockS3Trigger(bucket, key):
    event = dict(
            {
                "Records": [
                {
                    "s3": {
                    "bucket": {
                        "name": bucket
                    },
                    "object": {
                        "key": key
                    }
                    }
                }
                ]
            }
    )

    return event

parser = argparse.ArgumentParser(description='Process a file inS3 bucket')
parser.add_argument('-f', '--filename', dest='filename', required=True, help="file to process")
parser.add_argument('-b', '--bucket', dest='bucket', required=True, help="S3 bucket name")
args = parser.parse_args()

key = args.filename
bucket = args.bucket

event = mockS3Trigger(bucket, key)

results = lambda_handler(event, None)

print(f'Received: {results}')


from .base import DatabaseDownloader
import asyncpg
from fastapi import HTTPException, status

class PostgresDownloader(DatabaseDownloader):

    def __init__(self, connection_data:str, query:str, *args, **kwargs) -> None:
        super().__init__(connection_data, query, *args, **kwargs)

    async def get_data(self):
        """
        Получение данных из Postgresql
        """
        try:
            conn = await asyncpg.connect(self._connection_str)
            result = await conn.fetch(self.query)

            return [dict(row) for row in result]
        except Exception:
            raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f'Не удалось получить данные')


""" Tests for custom django template tags. """
from __future__ import absolute_import

import unittest

import ddt
import six

from django.template import Context, Template

@ddt.ddt
class OAExtrasTests(unittest.TestCase):
    """ Tests for custom django template tags oa_extras. """

    template = Template(
        u"{% load oa_extras %}"
        u"{{ text|link_and_linebreak }}"
    )

    @ddt.data(
        ("", ""),
        ('check this https://dummy-url.com', 'https://dummy-url.com'),
        ('Visit this URL http://dummy-url.com', 'http://dummy-url.com'),
        ('dummy-text http://dummy-url.org', 'http://dummy-url.org'),
        ('dummy-url.com dummy-text', 'dummy-url.com')
    )
    @ddt.unpack
    def test_link_and_linebreak(self, text, link_text):
        rendered_template = self.template.render(Context({'text': text}))
        self.assertIn(link_text, rendered_template)
        if text:
            six.assertRegex(
                self,
                rendered_template,
                r'{link_text}'.format(link_text=link_text),
            )

    @ddt.data(
        ("hello ", "script"),
        ("http://dummy-url.com <applet></applet>", "applet"),
        ("", "iframe"),
        ("<link></link>", "link"),
    )
    @ddt.unpack
    def test_html_tags(self, text, tag):
        rendered_template = self.template.render(Context({'text': text}))
        escaped_tag = "<{tag}>".format(tag=tag)
        self.assertIn(escaped_tag, rendered_template)


import collections as coll
import pandas as pd
import numpy as np
import sklearn.preprocessing as skpre
import sys

from FunctionBlock import FunctionBlock

# convert a dictionary of features to an array of doubles (in order)
def dictionaryToArray(dict, keys):
    result = []
    print('dict type = ', type(dict))
    print('keys in routine \n', keys)
    #print('tpe of keyes = ', keys.)
    for key in keys:   #.split(","):
        print('single key = ', key)
        print('dict = ', key, dict[key])
        result.append(dict[key])
    return result

def dotProduct(v1, v2):
    print('size v1 =', len(v1), )
    print('size v2 =', len(v2))
    print('v1.......\n', v1)
    print('v2.......\n', v2)
    print('zip.....\n', list(zip(v1,v2)))
    print('sum = ', sum([a*b for a,b in zip(v1, v2)]) )
    return [sum([a*b for a,b in zip(v1, v2)])]

class PLSTest(FunctionBlock):

    def __init__(self, name, unique_name):
        FunctionBlock.__init__(self, name, unique_name)

    def execute(self, results_table):
        try:
            FunctionBlock.report_status_executing(self)

            FunctionBlock.check_connector_has_one_wire(self, 'x')
            x_test_df = results_table[self.input_connectors['x'][0]]

            FunctionBlock.check_connector_has_one_wire(self, 'y')
            y_test_df = results_table[self.input_connectors['y'][0]]

            FunctionBlock.check_connector_has_one_wire(self, 'model')
            pls_model = results_table[self.input_connectors['model'][0]]
            #print('pls_model = \n', pls_model)
            coef_dict = {}
            for items in pls_model:
                coef_dict[items[0]] = items[1]

            #print('type coef_dict = ', type(coef_dict))
            #print('coef_dict \n', coef_dict)

            key_list = x_test_df.columns.values.tolist()
            #print('key_list \n', key_list)

            coef = dictionaryToArray(coef_dict, key_list)
            #print('coef = \n', coef)

            y_test_values = y_test_df.values
            # print('y test values........\n', y_test_values)
            # print('size of y test = ', y_test_values.size)

            x_test_values = x_test_df.values
            print('x test values.....\n', x_test_values)

            x_scaled = skpre.scale(x_test_values)
            y_scaled = skpre.scale(y_test_values)

            y_prediction = np.dot(coef, x_scaled.T)
            # print('y_prediction = ', y_prediction)
            # print('y_std = ', y_test_df.std())
            # print('y_mean = ', y_test_df.mean())

            # ss_error_total = sum((y_test_values - y_test_values.mean())**2)
            #
            # ss_error_residual = 0
            #
            # for x in range(len(y_test_values)):
            #     ss_error_residual += (y_test_values[x] - y_prediction[x])**2
            #
            # r2 = 1 - ss_error_residual/ss_error_total
            #
            # pls_test_result = coll.OrderedDict()
            # pls_test_result['r squared'] = r2[0]
            #
            # data_dict = {'y vals': list(y_test_values[:, 0]), 'y pred': list(y_prediction[:, 0])}

            #FunctionBlock.save_results(self, df=(pd.DataFrame(data_dict)), statistics=True, plot=True, results=pls_test_result)

            FunctionBlock.report_status_complete(self)

            return #{FunctionBlock.getFullPath(self, 'ycomp'): (pd.DataFrame(data_dict))}

        except Exception as err:
            FunctionBlock.save_results(self)
            FunctionBlock.report_status_failure(self)
            print(err.args, file=sys.stderr)


import json
import numpy as np
from tree import head_to_tree, tree_to_adj

filename="/data0/zhanglonghui/HATT-Proto/data/fewrel/val_fewrel_stf.json"
save_adj_path="/data0/zhanglonghui/HATT-Proto/data/fewrel/val_fewrel_adj.npy"
maxlen=47
prune=-1 #保留距离最短依存路径为prune以内的路径，-1表示保留所有路径

with open(filename) as infile:
    ori_data = json.load(infile)

def inputs_to_tree_reps(head, l, prune, subj_pos, obj_pos):
    trees = [head_to_tree(head[i], l[i], prune, subj_pos[i], obj_pos[i]) for i in range(len(l))]
    adj = [tree_to_adj(maxlen, tree, directed=False, self_loop=False).reshape(1, maxlen, maxlen) for tree in trees]
    adj = np.concatenate(adj, axis=0)
    return adj

def get_positions(start_idx, end_idx, length):
    """ Get subj/obj position sequence. """
    return list(range(-start_idx, 0)) + [0]*(end_idx - start_idx + 1) + \
            list(range(1, length-end_idx))

head, l ,subj_pos, obj_pos=[],[],[],[]
for relation in ori_data:
        for ex in ori_data[relation]:  # 遍历某个关系中的样本
            head.append([int(i) for i in ex["stanford_head"]]+[0]*(maxlen-len(ex["stanford_head"])))
            l.append(len(ex["tokens"]))
            subj_pos.append(get_positions(ex['t'][2][0][0],ex['t'][2][0][-1],len(ex["tokens"]))+[0]*(maxlen-len(ex["stanford_head"])))
            obj_pos.append(get_positions(ex['h'][2][0][0],ex['h'][2][0][-1], len(ex["tokens"])) + [0] * (maxlen - len(ex["stanford_head"])))

head, l ,subj_pos, obj_pos=[np.array(i) for i in [head, l ,subj_pos, obj_pos]]
adj = inputs_to_tree_reps(head, l, prune, subj_pos, obj_pos)

np.save(save_adj_path, adj)


# Size of variable arrays:
sizeAlgebraic = 25
sizeStates = 8
sizeConstants = 24
from math import *
from numpy import *

def createLegends():
    legend_states = [""] * sizeStates
    legend_rates = [""] * sizeStates
    legend_algebraic = [""] * sizeAlgebraic
    legend_voi = ""
    legend_constants = [""] * sizeConstants
    legend_voi = "time in component environment (millisecond)"
    legend_states[0] = "V in component membrane (millivolt)"
    legend_constants[0] = "R in component membrane (joule_per_kilomole_kelvin)"
    legend_constants[1] = "T in component membrane (kelvin)"
    legend_constants[2] = "F in component membrane (coulomb_per_mole)"
    legend_constants[3] = "C in component membrane (microF_per_cm2)"
    legend_algebraic[0] = "I_stim in component membrane (microA_per_cm2)"
    legend_algebraic[7] = "i_Na in component fast_sodium_current (microA_per_cm2)"
    legend_algebraic[15] = "i_si in component slow_inward_current (microA_per_cm2)"
    legend_algebraic[17] = "i_K in component time_dependent_potassium_current (microA_per_cm2)"
    legend_algebraic[21] = "i_K1 in component time_independent_potassium_current (microA_per_cm2)"
    legend_algebraic[23] = "i_Kp in component plateau_potassium_current (microA_per_cm2)"
    legend_algebraic[24] = "i_b in component background_current (microA_per_cm2)"
    legend_constants[4] = "stim_start in component membrane (millisecond)"
    legend_constants[5] = "stim_end in component membrane (millisecond)"
    legend_constants[6] = "stim_period in component membrane (millisecond)"
    legend_constants[7] = "stim_duration in component membrane (millisecond)"
    legend_constants[8] = "stim_amplitude in component membrane (microA_per_cm2)"
    legend_constants[9] = "g_Na in component fast_sodium_current (milliS_per_cm2)"
    legend_constants[18] = "E_Na in component fast_sodium_current (millivolt)"
    legend_constants[10] = "Nao in component ionic_concentrations (millimolar)"
    legend_constants[11] = "Nai in component ionic_concentrations (millimolar)"
    legend_states[1] = "m in component fast_sodium_current_m_gate (dimensionless)"
    legend_states[2] = "h in component fast_sodium_current_h_gate (dimensionless)"
    legend_states[3] = "j in component fast_sodium_current_j_gate (dimensionless)"
    legend_algebraic[1] = "alpha_m in component fast_sodium_current_m_gate (per_millisecond)"
    legend_algebraic[8] = "beta_m in component fast_sodium_current_m_gate (per_millisecond)"
    legend_algebraic[2] = "alpha_h in component fast_sodium_current_h_gate (per_millisecond)"
    legend_algebraic[9] = "beta_h in component fast_sodium_current_h_gate (per_millisecond)"
    legend_algebraic[3] = "alpha_j in component fast_sodium_current_j_gate (per_millisecond)"
    legend_algebraic[10] = "beta_j in component fast_sodium_current_j_gate (per_millisecond)"
    legend_algebraic[14] = "E_si in component slow_inward_current (millivolt)"
    legend_states[4] = "Cai in component intracellular_calcium_concentration (millimolar)"
    legend_states[5] = "d in component slow_inward_current_d_gate (dimensionless)"
    legend_states[6] = "f in component slow_inward_current_f_gate (dimensionless)"
    legend_algebraic[4] = "alpha_d in component slow_inward_current_d_gate (per_millisecond)"
    legend_algebraic[11] = "beta_d in component slow_inward_current_d_gate (per_millisecond)"
    legend_algebraic[5] = "alpha_f in component slow_inward_current_f_gate (per_millisecond)"
    legend_algebraic[12] = "beta_f in component slow_inward_current_f_gate (per_millisecond)"
    legend_constants[19] = "g_K in component time_dependent_potassium_current (milliS_per_cm2)"
    legend_constants[20] = "E_K in component time_dependent_potassium_current (millivolt)"
    legend_constants[12] = "PR_NaK in component time_dependent_potassium_current (dimensionless)"
    legend_constants[13] = "Ko in component ionic_concentrations (millimolar)"
    legend_constants[14] = "Ki in component ionic_concentrations (millimolar)"
    legend_states[7] = "X in component time_dependent_potassium_current_X_gate (dimensionless)"
    legend_algebraic[16] = "Xi in component time_dependent_potassium_current_Xi_gate (dimensionless)"
    legend_algebraic[6] = "alpha_X in component time_dependent_potassium_current_X_gate (per_millisecond)"
    legend_algebraic[13] = "beta_X in component time_dependent_potassium_current_X_gate (per_millisecond)"
    legend_constants[21] = "E_K1 in component time_independent_potassium_current (millivolt)"
    legend_constants[22] = "g_K1 in component time_independent_potassium_current (milliS_per_cm2)"
    legend_algebraic[20] = "K1_infinity in component time_independent_potassium_current_K1_gate (dimensionless)"
    legend_algebraic[18] = "alpha_K1 in component time_independent_potassium_current_K1_gate (per_millisecond)"
    legend_algebraic[19] = "beta_K1 in component time_independent_potassium_current_K1_gate (per_millisecond)"
    legend_constants[23] = "E_Kp in component plateau_potassium_current (millivolt)"
    legend_constants[15] = "g_Kp in component plateau_potassium_current (milliS_per_cm2)"
    legend_algebraic[22] = "Kp in component plateau_potassium_current (dimensionless)"
    legend_constants[16] = "E_b in component background_current (millivolt)"
    legend_constants[17] = "g_b in component background_current (milliS_per_cm2)"
    legend_rates[0] = "d/dt V in component membrane (millivolt)"
    legend_rates[1] = "d/dt m in component fast_sodium_current_m_gate (dimensionless)"
    legend_rates[2] = "d/dt h in component fast_sodium_current_h_gate (dimensionless)"
    legend_rates[3] = "d/dt j in component fast_sodium_current_j_gate (dimensionless)"
    legend_rates[5] = "d/dt d in component slow_inward_current_d_gate (dimensionless)"
    legend_rates[6] = "d/dt f in component slow_inward_current_f_gate (dimensionless)"
    legend_rates[7] = "d/dt X in component time_dependent_potassium_current_X_gate (dimensionless)"
    legend_rates[4] = "d/dt Cai in component intracellular_calcium_concentration (millimolar)"
    return (legend_states, legend_algebraic, legend_voi, legend_constants)

def initConsts():
    constants = [0.0] * sizeConstants; states = [0.0] * sizeStates;
    states[0] = -84.3801107371
    constants[0] = 8314
    constants[1] = 310
    constants[2] = 96484.6
    constants[3] = 1
    constants[4] = 100
    constants[5] = 9000
    constants[6] = 1000
    constants[7] = 2
    constants[8] = -25.5
    constants[9] = 23
    constants[10] = 140
    constants[11] = 18
    states[1] = 0.00171338077730188
    states[2] = 0.982660523699656
    states[3] = 0.989108212766685
    states[4] = 0.00017948816388306
    states[5] = 0.00302126301779861
    states[6] = 0.999967936476325
    constants[12] = 0.01833
    constants[13] = 5.4
    constants[14] = 145
    states[7] = 0.0417603108167287
    constants[15] = 0.0183
    constants[16] = -59.87
    constants[17] = 0.03921
    constants[18] = ((constants[0]*constants[1])/constants[2])*log(constants[10]/constants[11])
    constants[19] = 0.282000*(power(constants[13]/5.40000, 1.0/2))
    constants[20] = ((constants[0]*constants[1])/constants[2])*log((constants[13]+constants[12]*constants[10])/(constants[14]+constants[12]*constants[11]))
    constants[21] = ((constants[0]*constants[1])/constants[2])*log(constants[13]/constants[14])
    constants[22] = 0.604700*(power(constants[13]/5.40000, 1.0/2))
    constants[23] = constants[21]
    return (states, constants)

def computeRates(voi, states, constants):
    rates = [0.0] * sizeStates; algebraic = [0.0] * sizeAlgebraic
    algebraic[1] = (0.320000*(states[0]+47.1300))/(1.00000-exp(-0.100000*(states[0]+47.1300)))
    algebraic[8] = 0.0800000*exp(-states[0]/11.0000)
    rates[1] = algebraic[1]*(1.00000-states[1])-algebraic[8]*states[1]
    algebraic[2] = custom_piecewise([less(states[0] , -40.0000), 0.135000*exp((80.0000+states[0])/-6.80000) , True, 0.00000])
    algebraic[9] = custom_piecewise([less(states[0] , -40.0000), 3.56000*exp(0.0790000*states[0])+310000.*exp(0.350000*states[0]) , True, 1.00000/(0.130000*(1.00000+exp((states[0]+10.6600)/-11.1000)))])
    rates[2] = algebraic[2]*(1.00000-states[2])-algebraic[9]*states[2]
    algebraic[3] = custom_piecewise([less(states[0] , -40.0000), ((-127140.*exp(0.244400*states[0])-3.47400e-05*exp(-0.0439100*states[0]))*(states[0]+37.7800))/(1.00000+exp(0.311000*(states[0]+79.2300))) , True, 0.00000])
    algebraic[10] = custom_piecewise([less(states[0] , -40.0000), (0.121200*exp(-0.0105200*states[0]))/(1.00000+exp(-0.137800*(states[0]+40.1400))) , True, (0.300000*exp(-2.53500e-07*states[0]))/(1.00000+exp(-0.100000*(states[0]+32.0000)))])
    rates[3] = algebraic[3]*(1.00000-states[3])-algebraic[10]*states[3]
    algebraic[4] = (0.0950000*exp(-0.0100000*(states[0]-5.00000)))/(1.00000+exp(-0.0720000*(states[0]-5.00000)))
    algebraic[11] = (0.0700000*exp(-0.0170000*(states[0]+44.0000)))/(1.00000+exp(0.0500000*(states[0]+44.0000)))
    rates[5] = algebraic[4]*(1.00000-states[5])-algebraic[11]*states[5]
    algebraic[5] = (0.0120000*exp(-0.00800000*(states[0]+28.0000)))/(1.00000+exp(0.150000*(states[0]+28.0000)))
    algebraic[12] = (0.00650000*exp(-0.0200000*(states[0]+30.0000)))/(1.00000+exp(-0.200000*(states[0]+30.0000)))
    rates[6] = algebraic[5]*(1.00000-states[6])-algebraic[12]*states[6]
    algebraic[6] = (0.000500000*exp(0.0830000*(states[0]+50.0000)))/(1.00000+exp(0.0570000*(states[0]+50.0000)))
    algebraic[13] = (0.00130000*exp(-0.0600000*(states[0]+20.0000)))/(1.00000+exp(-0.0400000*(states[0]+20.0000)))
    rates[7] = algebraic[6]*(1.00000-states[7])-algebraic[13]*states[7]
    algebraic[14] = 7.70000-13.0287*log(states[4]/1.00000)
    algebraic[15] = 0.0900000*states[5]*states[6]*(states[0]-algebraic[14])
    rates[4] = (-0.000100000/1.00000)*algebraic[15]+0.0700000*(0.000100000-states[4])
    algebraic[0] = custom_piecewise([greater_equal(voi , constants[4]) & less_equal(voi , constants[5]) & less_equal((voi-constants[4])-floor((voi-constants[4])/constants[6])*constants[6] , constants[7]), constants[8] , True, 0.00000])
    algebraic[7] = constants[9]*(power(states[1], 3.00000))*states[2]*states[3]*(states[0]-constants[18])
    algebraic[16] = custom_piecewise([greater(states[0] , -100.000), (2.83700*(exp(0.0400000*(states[0]+77.0000))-1.00000))/((states[0]+77.0000)*exp(0.0400000*(states[0]+35.0000))) , True, 1.00000])
    algebraic[17] = constants[19]*states[7]*algebraic[16]*(states[0]-constants[20])
    algebraic[18] = 1.02000/(1.00000+exp(0.238500*((states[0]-constants[21])-59.2150)))
    algebraic[19] = (0.491240*exp(0.0803200*((states[0]+5.47600)-constants[21]))+1.00000*exp(0.0617500*(states[0]-(constants[21]+594.310))))/(1.00000+exp(-0.514300*((states[0]-constants[21])+4.75300)))
    algebraic[20] = algebraic[18]/(algebraic[18]+algebraic[19])
    algebraic[21] = constants[22]*algebraic[20]*(states[0]-constants[21])
    algebraic[22] = 1.00000/(1.00000+exp((7.48800-states[0])/5.98000))
    algebraic[23] = constants[15]*algebraic[22]*(states[0]-constants[23])
    algebraic[24] = constants[17]*(states[0]-constants[16])
    rates[0] = (-1.00000/constants[3])*(algebraic[0]+algebraic[7]+algebraic[15]+algebraic[17]+algebraic[21]+algebraic[23]+algebraic[24])
    return(rates)

def computeAlgebraic(constants, states, voi):
    algebraic = array([[0.0] * len(voi)] * sizeAlgebraic)
    states = array(states)
    voi = array(voi)
    algebraic[1] = (0.320000*(states[0]+47.1300))/(1.00000-exp(-0.100000*(states[0]+47.1300)))
    algebraic[8] = 0.0800000*exp(-states[0]/11.0000)
    algebraic[2] = custom_piecewise([less(states[0] , -40.0000), 0.135000*exp((80.0000+states[0])/-6.80000) , True, 0.00000])
    algebraic[9] = custom_piecewise([less(states[0] , -40.0000), 3.56000*exp(0.0790000*states[0])+310000.*exp(0.350000*states[0]) , True, 1.00000/(0.130000*(1.00000+exp((states[0]+10.6600)/-11.1000)))])
    algebraic[3] = custom_piecewise([less(states[0] , -40.0000), ((-127140.*exp(0.244400*states[0])-3.47400e-05*exp(-0.0439100*states[0]))*(states[0]+37.7800))/(1.00000+exp(0.311000*(states[0]+79.2300))) , True, 0.00000])
    algebraic[10] = custom_piecewise([less(states[0] , -40.0000), (0.121200*exp(-0.0105200*states[0]))/(1.00000+exp(-0.137800*(states[0]+40.1400))) , True, (0.300000*exp(-2.53500e-07*states[0]))/(1.00000+exp(-0.100000*(states[0]+32.0000)))])
    algebraic[4] = (0.0950000*exp(-0.0100000*(states[0]-5.00000)))/(1.00000+exp(-0.0720000*(states[0]-5.00000)))
    algebraic[11] = (0.0700000*exp(-0.0170000*(states[0]+44.0000)))/(1.00000+exp(0.0500000*(states[0]+44.0000)))
    algebraic[5] = (0.0120000*exp(-0.00800000*(states[0]+28.0000)))/(1.00000+exp(0.150000*(states[0]+28.0000)))
    algebraic[12] = (0.00650000*exp(-0.0200000*(states[0]+30.0000)))/(1.00000+exp(-0.200000*(states[0]+30.0000)))
    algebraic[6] = (0.000500000*exp(0.0830000*(states[0]+50.0000)))/(1.00000+exp(0.0570000*(states[0]+50.0000)))
    algebraic[13] = (0.00130000*exp(-0.0600000*(states[0]+20.0000)))/(1.00000+exp(-0.0400000*(states[0]+20.0000)))
    algebraic[14] = 7.70000-13.0287*log(states[4]/1.00000)
    algebraic[15] = 0.0900000*states[5]*states[6]*(states[0]-algebraic[14])
    algebraic[0] = custom_piecewise([greater_equal(voi , constants[4]) & less_equal(voi , constants[5]) & less_equal((voi-constants[4])-floor((voi-constants[4])/constants[6])*constants[6] , constants[7]), constants[8] , True, 0.00000])
    algebraic[7] = constants[9]*(power(states[1], 3.00000))*states[2]*states[3]*(states[0]-constants[18])
    algebraic[16] = custom_piecewise([greater(states[0] , -100.000), (2.83700*(exp(0.0400000*(states[0]+77.0000))-1.00000))/((states[0]+77.0000)*exp(0.0400000*(states[0]+35.0000))) , True, 1.00000])
    algebraic[17] = constants[19]*states[7]*algebraic[16]*(states[0]-constants[20])
    algebraic[18] = 1.02000/(1.00000+exp(0.238500*((states[0]-constants[21])-59.2150)))
    algebraic[19] = (0.491240*exp(0.0803200*((states[0]+5.47600)-constants[21]))+1.00000*exp(0.0617500*(states[0]-(constants[21]+594.310))))/(1.00000+exp(-0.514300*((states[0]-constants[21])+4.75300)))
    algebraic[20] = algebraic[18]/(algebraic[18]+algebraic[19])
    algebraic[21] = constants[22]*algebraic[20]*(states[0]-constants[21])
    algebraic[22] = 1.00000/(1.00000+exp((7.48800-states[0])/5.98000))
    algebraic[23] = constants[15]*algebraic[22]*(states[0]-constants[23])
    algebraic[24] = constants[17]*(states[0]-constants[16])
    return algebraic

def custom_piecewise(cases):
    """Compute result of a piecewise function"""
    return select(cases[0::2],cases[1::2])

def solve_model():
    """Solve model with ODE solver"""
    from scipy.integrate import ode
    # Initialise constants and state variables
    (init_states, constants) = initConsts()

    # Set timespan to solve over
    voi = linspace(0,500,500)

    # Construct ODE object to solve
    r = ode(computeRates)
    r.set_integrator('vode', method='bdf', atol=1e-06, rtol=1e-06, max_step=1)
    r.set_initial_value(init_states, voi[0])
    r.set_f_params(constants)

    # Solve model
    states = array([[0.0] * len(voi)] * sizeStates)
    states[:,0] = init_states
    for (i,t) in enumerate(voi[1:]):
        if r.successful():
            r.integrate(t)
            states[:,i+1] = r.y
        else:
            break

    # Compute algebraic variables
    algebraic = computeAlgebraic(constants, states, voi)
    return (voi, states, algebraic)

def plot_model(voi, states, algebraic):
    """Plot variables against variable of integration"""
    import pylab
    (legend_states, legend_algebraic, legend_voi, legend_constants) = createLegends()
    pylab.figure(1)
    pylab.plot(voi,states[0])
    pylab.xlabel(legend_voi)
    #pylab.legend(legend_states + legend_algebraic, loc='best')
    pylab.show()

if __name__ == "__main__":
    (voi, states, algebraic) = solve_model()
    plot_model(voi, states, algebraic)


from django.shortcuts import render, redirect
from django.http import HttpResponse
from django import forms
from random import randint
from . import util

def index(request):
    return render(request, "encyclopedia/index.html", {
        "entries": util.list_entries()
    })

def get_title(request, title):
    """
    Get content of the encyclopedia entry per requierment 2
    """
    return render(request, "encyclopedia/title.html", {
        "title_name": title,
        "title": util.get_entry(title)
    })

class EditForm(forms.Form):
    """
    Define form used to edit entries per requierment 16 of readme
    """
    content = forms.CharField(widget=forms.Textarea,
                              max_length=1000, required=True)

def edit(request, title_name):
    """
    Edit the contents of a title per requierments 16 of readme
    """
    # Listen for a POST request and process data when detected
    if request.method == "POST":
        form = EditForm(request.POST)
        if form.is_valid():
            edited_content = form.cleaned_data['content']
            util.save_entry(title_name, edited_content)
            return redirect(f"/wiki/{ title_name }")

    # get content from file
    else:
        content = util.get_entry(title_name, return_html=False)
        return render(request, "encyclopedia/edit.html", {
            "form": EditForm({"content": content}),
            "title_name": title_name
            })

class NewEntryForm(forms.Form):
    """
    Define form used to create new entry per requierment 11 of readme
    """
    title = forms.CharField(max_length=50, required=True)
    content = forms.CharField(widget=forms.Textarea,
                              max_length=1000, required=True)

def new(request):
    """
    Handle a new entry per requierment 11 of readme
    """
    # Listen for a POST request and process data when detected
    if request.method == "POST":
        form = NewEntryForm(request.POST)
        if form.is_valid():
            title = form.cleaned_data['title']
            content = form.cleaned_data['content']

            # standardize content with title header
            content = f"#{title}\n\n{content}"
            # standardize title for file name
            title = util.clean_title(form.cleaned_data['title'])

            # If the title is unique per requierment 14 in readme
            if title not in util.list_entries():
                util.save_entry(title, content)
                # open newly created page per requierment 15 of readme
                return get_title(request, title)
            else:
                return render(request, "encyclopedia/error.html")

    # Else create a blank form
    else:
        return render(request, "encyclopedia/new.html", {
            "form": NewEntryForm()
            })

def random(request):
    """
    Get a random page per requierment 20 of readme
    """
    entries = util.list_entries()
    n = randint(0, len(entries)-1)
    target_entry = entries[n]
    return redirect(f"/wiki/{ target_entry }")

def search(request):
    """
    Get contents of the search per requierments 7, 8, 9, 10
    """
    query = request.GET.get('q')
    results = util.search(query)

    # if an exact match open that entry
    if len(results) == 1:
        return redirect(f"/wiki/{ results[0] }")

    # if multiple partial matches open list of options
    elif len(results) > 1:
        return render(request, "encyclopedia/search_results.html", {
            "entries": results
        })
    # else prompt user for new input
    else:
        return render(request, "encyclopedia/search_empty.html")


import requests
from bs4 import BeautifulSoup
import lxml
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
import time

URL = "https://www.zillow.com/homes/for_rent/1-_beds/?searchQueryState=%7B%22pagination%22%3A%7B%7D%2C%22mapBounds%22%3A%7B%22west%22%3A-122.56825484228516%2C%22east%22%3A-122.29840315771484%2C%22south%22%3A37.69234177970014%2C%22north%22%3A37.85814816331502%7D%2C%22mapZoom%22%3A12%2C%22isMapVisible%22%3Afalse%2C%22filterState%22%3A%7B%22price%22%3A%7B%22max%22%3A872627%7D%2C%22beds%22%3A%7B%22min%22%3A1%7D%2C%22fore%22%3A%7B%22value%22%3Afalse%7D%2C%22mp%22%3A%7B%22max%22%3A3000%7D%2C%22auc%22%3A%7B%22value%22%3Afalse%7D%2C%22nc%22%3A%7B%22value%22%3Afalse%7D%2C%22fr%22%3A%7B%22value%22%3Atrue%7D%2C%22fsbo%22%3A%7B%22value%22%3Afalse%7D%2C%22cmsn%22%3A%7B%22value%22%3Afalse%7D%2C%22fsba%22%3A%7B%22value%22%3Afalse%7D%7D%2C%22isListVisible%22%3Atrue%7D"

headers = {
    "User-Agent": USER-AGENT,
    "Accept-Language": "en-US,en;q=0.9"
}

response = requests.get(URL, headers=headers)
webpage = response.text

soup = BeautifulSoup(webpage, "lxml")

price_list = []
for price in soup.find_all(name="div", class_="kJFQQX"):
    price_list.append(price.getText()[:6])

link_list = []
for link in soup.find_all(name="div", class_="clrcsE"):
    new_link = link.find('a')["href"]
    if new_link[0] != 'h':
        new_link = "https://www.zillow.com" + new_link
    link_list.append(new_link)

address_list = []
for address in soup.find_all(name="div", class_="bDmKKM"):
    new_address = address.find('address').getText()
    address_list.append(new_address)

chrome_driver_path = "C:/Development/chromedriver.exe"
ser = Service(chrome_driver_path)

chrome_options = Options()
chrome_options.add_experimental_option("detach", True)

driver = webdriver.Chrome(service=ser, options=chrome_options)

for i in range(len(price_list)):
    driver.get(SHEET_URL)
    time.sleep(2)

    answer_1 = driver.find_element(by="xpath", value='//*[@id="mG61Hd"]/div[2]/div/div[2]/div[1]/div/div/div[2]/div/div[1]/div/div[1]/input')
    answer_1.send_keys(address_list[i])

    answer_2 = driver.find_element(by="xpath", value='//*[@id="mG61Hd"]/div[2]/div/div[2]/div[2]/div/div/div[2]/div/div[1]/div/div[1]/input')
    answer_2.send_keys(price_list[i])

    answer_3 = driver.find_element(by="xpath", value='//*[@id="mG61Hd"]/div[2]/div/div[2]/div[3]/div/div/div[2]/div/div[1]/div/div[1]/input')
    answer_3.send_keys(link_list[i])

    submit_button = driver.find_element(by="xpath", value='//*[@id="mG61Hd"]/div[2]/div/div[3]/div[1]/div[1]/div/span/span')
    submit_button.click()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals

__author__ = 'Sander van Rijn <svr003@gmail.com>'

import json
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import os
import pprint
import cPickle
from collections import Counter, namedtuple
from datetime import timedelta
from modea.Utils import getPrintName, getOpts, intToRepr, reprToInt, reprToString
from local import brute_location, ga_location, raw_bfname, raw_ganame

np.set_printoptions(linewidth=156)

dimensions = [2]#,3,5], 10, 20]
functions = range(1,2)
runs=range(8,9)
subgroups = [
    [1,  2,  3,  4,  5],
    [6,  7,  8,  9],
    [10, 11, 12, 13, 14],
    [15, 16, 17, 18, 19],
    [20, 21, 22, 23, 24]
]
np_save_names = ['time_spent', 'generation_sizes', 'sigma', 'best_result', 'best_fitness']

default_ESs = [
    reprToInt([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),  # CMA_ES                          6
    reprToInt([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),  # active CMA_ES                   5
    reprToInt([0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),  # elitist CMA_ES                 14
    reprToInt([0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0]),  # mirrored-pairwise CMA_ES        9
    reprToInt([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]),  # IPOP CMA_ES                    25
    reprToInt([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]),  # active-IPOP CMA_ES              5
    reprToInt([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1]),  # elitist active-IPOP CMA_ES     14
    reprToInt([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]),  # BIPOP CMA_ES                   19
    reprToInt([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]),  # active-BIPOP CMA_ES             9
    reprToInt([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2]),  # elitist active-BIPOP CMA_ES    14
]

ES_and_result = namedtuple('ES_and_result', ['ES', 'fitness'])
ranked_ES_and_result = namedtuple('ranked_ES_and_result', ['ES', 'rank', 'fitness'])

### Utility functions ###
def tdatFileToFitnesses(filename):
    """
        Given a brute_force filename, load all associated ESFitness objects and return them in tuples with both an
        numerical and string value of the relevant ES-structure. N.B.: the filename is **not** checked for correctness

        :param filename:    A string expected to be in the format: 'data\\<experiment_name>_{}_f{}.tdat'
        :return:            List of tuples (reprToInt(ES), reprToString(ES), ESFitness), sorted by ESFitness
    """

    results = []
    with open(filename) as f:
        for line in f:
            parts = line.split('\t')
            results.append(ES_and_result(eval(parts[0]), eval(parts[1])))
    return results

### GA STUFF ###
def storeBestFromGA():
    os.chdir(ga_location)
    results = {dim: {} for dim in dimensions}
    for dim in dimensions:
        for fid in functions:

            ga_results = tdatFileToFitnesses(raw_ganame.format(dim, fid))
            ga_results.sort(key=lambda a: a.fitness)
            results[dim][fid] = ga_results[0]

    with open('ga_results.dat', 'w') as f:
        cPickle.dump(results, f)

def getBestEs():
    os.chdir(ga_location)
    results = {dim: {} for dim in dimensions}
    for dim in dimensions:
        results[dim] = {func: {} for func in functions}
        # for file in final_files:
        for fid in functions:
            results[dim][fid] = {run: {} for run in runs}
            for run in runs:
                filename = 'C:\\Users\\jelle laptop\\Desktop\\good_results_2_3\\final_GA_results_{}dim_f{}_run{}.npz'.format(dim, fid,run)
                x = np.load(filename)
                # for data in ['time_spent', 'best_result']:
                for data in x.files:  # ['time_spent', 'generation_sizes', 'sigma', 'best_result', 'best_fitness']
                    results[dim][fid][run][data] = x[data]
    return results

def storeResults():
    results = getBestEs()
    np.savez('final_GA_results.npz', results=results, dims=dimensions, functions=functions, np_save_names=np_save_names)

def printResults():

    os.chdir(ga_location)
    x = np.load('final_GA_results.npz')
    results = x['results'].item()
    for dim in dimensions:
        print("{}-dimensional:".format(dim))
        for func in functions:
            print("  F{}:\t{} {}".format(func, results[dim][func]['best_result'], getPrintName(getOpts(results[dim][func]['best_result']))))

def createGARunPlots():
    os.chdir(ga_location)
    x = np.load('final_GA_results.npz')
    results = x['results'].item()

    matplotlib.rcParams.update({'font.size': 14})
    plt.figure(figsize=(8,4.5))

    for func in functions:
        print("F{}:".format(func))

        plt.clf()
        for dim in dimensions:
            # best_found_ever_FCE
            best_found_ever_FCE=[0]*30
            best_found_ever_ERT=[0]*30
            for run in runs:
                best_per_generation = results[dim][func][run]['best_fitness'][::12]
                best_found_ever_inrun = []
                for i, fit in enumerate(best_per_generation):
                    if fit <= min(best_per_generation[:i+1]):
                        best_found_ever_inrun.append(fit)
                    else:
                        best_found_ever_inrun.append(best_found_ever_inrun[i-1])
                temp_num=0
                for p, x in enumerate(best_found_ever_inrun):
                    # best_found_ever[x]+=best_found_ever_inrun[x]
                    # print("Dit is duidelijk x",x)
                    # print("p,x::",p,x.FCE, x.ERT)
                    best_found_ever_FCE[p]+=x.FCE
                    if x.ERT is not None:
                        best_found_ever_ERT[p]+=x.ERT
                    else:
                        best_found_ever_ERT[p]+= 32 * 1e3 * dim
                        # print("x.ERT is None")
                        # temp_num=temp_num+1
                    # best_found_ever[p].FCE=
                    # print("FCE!!!::::",p , x.FCE)
                    # print("ERT!!!::::",p , x.ERT)
                    # print(best_found_ever_inrun[x])
                    # print("/n")

            # print("FCE!:", best_found_ever_FCE)
            # print("ERT!:", best_found_ever_ERT)
            # aggregate_fitnesses = []
            # ggregate_fitnesses = []

            # for index in len(runs*dim):
            #     default_ERT = 32 * 1e3 * dim
            #     ERT = 0
            #     FCE = 0
            #     for run in runs:
            #         x = best_found_ever[run][index]
            #         FCE += fit.FCE
            #         ERT += fit.ERT if fit.ERT is not None else default_ERT
            #     aggregate_fitnesses.append(ESFitness(ERT=ERT/runs, FCE=FCE/runs))

            # print("temp_num",temp_num)
            best_found_ever_FCE[:] = [x / len(runs) for x in best_found_ever_FCE]
            best_found_ever_ERT[:] = [x / len(runs)-temp_num for x in best_found_ever_ERT]
            # print("FCE2!:", best_found_ever_FCE)
            # print("ERT2!:", best_found_ever_ERT)

            plt.subplot(1, 2, 1)
            plt.plot(best_found_ever_FCE, label='{}-dim'.format(dim))
            plt.subplot(1, 2, 2)
            plt.plot(best_found_ever_ERT, label='{}-dim'.format(dim))

        plt.suptitle("Convergence for F{}".format(func), y=.99)

        plt.subplot(1, 2, 1)
        plt.yscale('log')
        plt.xlabel('Generation')
        plt.ylabel('FCE')
        plt.legend(loc=0, prop={'size':11}, labelspacing=0.15)

        plt.subplot(1, 2, 2)
        plt.yscale('log')
        plt.xlabel('Generation')
        plt.ylabel('ERT')
        plt.legend(loc=0, prop={'size':11}, labelspacing=0.15)

        plt.tight_layout()

        plt.savefig('img/F{}_log.png'.format(func), bbox_inches='tight')
        # plt.savefig('img/F{}_log.pdf'.format(func), bbox_inches='tight')

def createGARunPlots_v2():
        os.chdir(ga_location)
        x = np.load('final_GA_results.npz')
        results = x['results'].item()

        matplotlib.rcParams.update({'font.size': 14})
        plt.figure(figsize=(8, 4.5))

        for func in functions:
            print("F{}:".format(func))

            for dim in dimensions:
                for run in runs:
                    best_per_generation = results[dim][func][run]['best_fitness'][::12]
                    best_found_ever = []
                    for i, fit in enumerate(best_per_generation):
                        if fit <= min(best_per_generation[:i + 1]):
                            best_found_ever.append(fit)
                        else:
                            best_found_ever.append(best_found_ever[i - 1])
                    plt.clf()

                    plt.subplot(1, 2, 1)
                    plt.plot([x.FCE for x in best_found_ever], label='{}-dim'.format(dim))
                    plt.subplot(1, 2, 2)
                    plt.plot([x.ERT for x in best_found_ever], label='{}-dim'.format(dim))

                    plt.suptitle("Convergence for F{}".format(func), y=.99)

                    plt.subplot(1, 2, 1)
                    plt.yscale('log')
                    plt.xlabel('Generation')
                    plt.ylabel('FCE')
                    plt.legend(loc=0, prop={'size': 11}, labelspacing=0.15)

                    plt.subplot(1, 2, 2)
                    plt.yscale('log')
                    plt.xlabel('Generation')
                    plt.ylabel('ERT')
                    plt.legend(loc=0, prop={'size': 11}, labelspacing=0.15)

                    plt.tight_layout()

                    plt.savefig('img/F{}_R{}_log.png'.format(func, run), bbox_inches='tight')
                    # plt.savefig('img/F{}_log.pdf'.format(func), bbox_inches='tight')

def printTable(results):
    print(results[2][1][8]['alpha_mu'])
    print('\\hline')
    print('F-ID & N & GA & FCE & ERT \\\\')
    print('\\hline')
    print('\\hline')
    for fid in functions:
        for dim in dimensions:
            for run in runs:
                result, fit = results[dim][fid][run]
                # print("fit:",fit)
                string = ''
                # for i in range(len(result)):
                for i in range(26):
                    a = round(float(result[i]),3)
                    string += ' '
                    string += str(a)
                # for i in range (12,26):
                #     a = round(int(result[i]), 3)
                #     string += ' '
                #     string += str(a)

                print('F{0} & {1} & {2} & {3:.3g} & {4}\\\\'.format(fid, dim, string, fit.FCE, fit.ERT))
        print('\\hline')

def printTable_v2(results):
    print('\\hline')
    print('F-ID & N & GA & FCE & ERT \\\\')
    print('\\hline')
    print('\\hline')
    for fid in functions:
        for dim in dimensions:
            for run in runs:
                result, fit = results[dim][fid][run]
                string = ''
                # for i in range(len(result)):
                for i in range(11):
                    string += str(result[i])

                print('F{0} & {1} & {2} & {3:.3g} & {4}\\\\'.format(fid, dim, string, fit.FCE, fit.ERT))
        print('\\hline')

def printGATable():

    os.chdir(ga_location)
    x = np.load('final_GA_results.npz')
    results = x['results'].item()
    ga_results = {dim:
                      {fid:
                           {run:
                                (results[dim][fid][run]['best_result'], min(results[dim][fid][run]['best_fitness']))for run in runs} for fid in functions} for dim in dimensions}
    # print(round((results[2][1][5]['best_result'])

    printTable(ga_results)

def printGAcount():
    os.chdir(ga_location)
    x = np.load('final_GA_results.npz')
    results = x['results'].item()
    ga_results = {dim: {fid: (results[dim][fid]['best_result'], min(results[dim][fid]['best_fitness'])) for fid in functions} for dim in dimensions}
    printIntCount(ga_results)

def storeRepresentation():

    os.chdir(ga_location)
    x = np.load('final_GA_results.npz')
    results = x['results'].item()

    to_store = {dim: {} for dim in dimensions}
    for dim in dimensions:
        print("{}-dimensional:".format(dim))
        for func in functions:
            to_store[dim][func] = results[dim][func]['best_result'].tolist()
            print("  F{}:\t{}".format(func, results[dim][func]['best_result']))

    pprint.pprint(to_store)
    with open('ES_per_experiment.json', 'w') as json_out:
        json.dump(to_store, json_out)

def GAtimeSpent():
    os.chdir(ga_location)
    times = []
    for dim in dimensions:
        for fid in functions:
            filename = 'final_stats\\final_GA_results_{}dim_f{}.npz'.format(dim, fid)
            x = np.load(filename)
            times.append(x['time_spent'])
    print(min(times), max(times), sum(times, timedelta(0)) // len(times))

def printCompTable(bf, ga):
    print('\\hline')
    print('F-ID & N & Brute Force & FCE & ERT & GA & FCE & ERT \\\\')
    print('\\hline')
    print('\\hline')
    for fid in functions:
        for dim in dimensions:
            bf_result, bf_fit = bf[dim][fid]
            ga_result, ga_fit = ga[dim][fid]
            ga_diff = ''
            bf_string = ''
            for i in range(len(bf_result)):
                bf_string += str(bf_result[i])
                if bf_result[i] != ga_result[i]:
                    ga_diff += '\\underline{}{}{}'.format('{', ga_result[i], '}')
                else:
                    ga_diff += str(ga_result[i])

            bf_ert = np.inf if bf_fit.ERT is None else bf_fit.ERT
            ga_ert = np.inf if ga_fit.ERT is None else ga_fit.ERT
            print('F{0} & {1} & {2} & {3:.3g} & {4:.3g} & {5} & {6:.3g} & {7:.3g}\\\\'.format(
                fid, dim, bf_string, bf_fit.FCE, bf_ert, ga_diff, ga_fit.FCE, ga_ert
            ))
        print('\\hline')

### BF STUFF ###
def checkFileSizesBF():
    os.chdir(brute_location)

    for dim in dimensions:
        print(dim)
        for fid in functions:
            with open(raw_bfname.format(dim, fid)) as f:
                lines = [line for line in f]
                if len(lines) != 4608:
                    print("File bruteforce_{}_f{}.tdat does not contain 4608 entries! ({})".format(dim, fid, len(lines)))

def storeBestFromBF():
    os.chdir(brute_location)
    results = {dim: {} for dim in dimensions}
    for dim in dimensions:
        for fid in functions:

            bf_results = tdatFileToFitnesses(raw_bfname.format(dim, fid))
            bf_results.sort(key=lambda a: a.fitness)
            results[dim][fid] = bf_results[0]

    with open('brute_results.dat', 'w') as f:
        cPickle.dump(results, f)

def printBFFitDistances():
    os.chdir(brute_location)
    for dim in dimensions:
        for fid in functions:
            bf_results = tdatFileToFitnesses(raw_bfname.format(dim, fid))
            bf_results.sort(key=lambda a: a.fitness)
            print("{:>2}dim F{:>2}: {}".format(dim, fid, [str(res.fitness) for res in bf_results[::100]]))

def findGAInRankedBF():
    os.chdir(ga_location)
    x = np.load('final_GA_results.npz')
    ga_results = x['results'].item()

    results = {dim: {} for dim in dimensions}
    os.chdir(brute_location)
    for dim in dimensions:
        for fid in functions:
            ga = reprToInt(ga_results[dim][fid]['best_result'])
            fit = ga_results[dim][fid]['best_fitness'][-1]

            bf_results = tdatFileToFitnesses(raw_bfname.format(dim, fid))
            bf_results.sort(key=lambda a: a.fitness)
            indexes = [reprToInt(a.ES) for a in bf_results]
            ga_index = indexes.index(ga)

            # Where does the GA-found ERT/FCE result rank in the brute-force results?
            fit_index = 0
            max_index = len(bf_results)
            while fit_index < max_index and fit > bf_results[fit_index].fitness:
                fit_index += 1

            results[dim][fid] = (ga, fit_index, ga_index, indexes)
            print("{:>2}D F{:>2}:  GA {:>4} is ranked {:>4} ({:>4})\t\t\t GA: {} \t BF[0]: {}".format(dim, fid, ga,
                                                                                fit_index, ga_index,
                                                                                fit,
                                                                                bf_results[0].fitness))

    with open('rank_ga_in_bf.dat', 'w') as f:
        cPickle.dump(results, f)

def printGAInRankedBF():

    os.chdir(brute_location)
    with open('rank_ga_in_bf.dat') as f:
        results = cPickle.load(f)

    fit_ranks = []
    ga_ranks = []
    for dim in dimensions:
        for fid in functions:
            fit_ranks.append(results[dim][fid][1])
            ga_ranks.append(results[dim][fid][2])

    count = Counter(fit_ranks)
    fit_ranking = sorted(count.items(), key=lambda x: x[0])
    fit_ranking.reverse()
    count = Counter(ga_ranks)
    struct_ranking = sorted(count.items(), key=lambda x: x[0])
    struct_ranking.reverse()

    f = s = 0
    full_ranking = []
    while len(fit_ranking) > 0 or len(struct_ranking) > 0:
        if len(fit_ranking) > 0:
            fit_rank, fit_count = fit_ranking[-1]
        else:
            fit_rank = fit_count = 1e5
        str_rank, str_count = struct_ranking[-1]

        if fit_rank == str_rank:
            full_ranking.append((fit_rank, fit_count, str_count))
            fit_ranking.pop()
            struct_ranking.pop()
        elif fit_rank < str_rank:
            full_ranking.append((fit_rank, fit_count, 0))
            fit_ranking.pop()
        elif str_rank < fit_rank:
            full_ranking.append((str_rank, 0, str_count))
            struct_ranking.pop()

    for rank, f_count, s_count in full_ranking:
        print("{:>4} & {:>2} & {:>2} \\\\".format(rank+1, f_count, s_count))

def printBFTable():

    os.chdir(brute_location)
    with open('brute_results.dat') as f:
        x = cPickle.load(f)
        printTable(x)

def printDoubleTable():

    os.chdir(ga_location)
    x = np.load('final_GA_results.npz')
    results = x['results'].item()
    ga_results = {dim: {fid: (results[dim][fid]['best_result'], min(results[dim][fid]['best_fitness'])) for fid in functions} for dim in dimensions}

    os.chdir(brute_location)
    with open('brute_results.dat') as f:
        bf_results = cPickle.load(f)
    printCompTable(bf_results, ga_results)

def printIntCount(results, fids=None, dims=None):

    if fids is None:
        fids = functions
    if dims is None:
        dims = dimensions

    from collections import Counter

    all_strings = []
    for fid in fids:
        for dim in dims:
            all_strings.append(results[dim][fid][0])

    choice_lists = zip(*all_strings)
    counters = [Counter(int_list) for int_list in choice_lists]
    # print(counters)
    # print()
    for count in counters[:11]:
        n = (count[0] + count[1] + count[2]) / 100
        print("{:>5.3} {:>5.3} {:>5.3}".format(count[0]/n, count[1]/n, count[2]/n))
        # for i in range(3):
        #     print(count[i], ' ', end='')
        # print()

def printDoubleCount(fids=None, dims=None):

    if fids is None:
        fids = functions
    if dims is None:
        dims = dimensions

    os.chdir(ga_location)
    x = np.load('final_GA_results.npz')
    results = x['results'].item()
    ga_results = {dim: {fid: (results[dim][fid]['best_result'], min(results[dim][fid]['best_fitness'])) for fid in fids} for dim in dims}
    printIntCount(ga_results, fids, dims)
    print()
    os.chdir(brute_location)
    with open('brute_results.dat') as f:
        bf_results = cPickle.load(f)
    printIntCount(bf_results, fids, dims)

def findGivenInRankedBF(dim, fid, given):
    os.chdir(brute_location)
    bf_results = tdatFileToFitnesses(raw_bfname.format(dim, fid))
    bf_results.sort(key=lambda a: a.fitness)
    indexes = [reprToInt(a.ES) for a in bf_results]

    results = []
    for ES in given:
        index = indexes.index(ES)
        results.append(ranked_ES_and_result(ES, index, bf_results[index].fitness))
    return results

def getBestFromRankedBF(dim, fid, num=10):
    os.chdir(brute_location)
    bf_results = tdatFileToFitnesses(raw_bfname.format(dim, fid))
    bf_results.sort(key=lambda a: a.fitness)
    indexes = [reprToInt(a.ES) for a in bf_results]

    results = []
    for i in range(num):
        results.append((indexes[i], i, bf_results[i].fitness))

    return results

def printBestFromRankedBF():
    for dim in dimensions:
        for fid in functions:
            print("Results for F{} in {}dim:".format(fid, dim))
            # print(findGivenInRankedBF(dim, 1, given))
            results = getBestFromRankedBF(dim, fid, num=10)
            for ES, rank, fit in results:
                print("Rank: {0:>4}\t{1:>33}\t{2}".format(rank+1, intToRepr(ES), fit))
            print()

def correlationMatrix(fids=None):

    if fids is None:
        fids = functions

    os.chdir(ga_location)
    x = np.load('final_GA_results.npz')
    results = x['results'].item()
    ga_results = {dim: {fid: results[dim][fid]['best_result'] for fid in fids} for dim in dimensions}

    os.chdir(brute_location)
    with open('brute_results.dat') as f:
        bf_all = cPickle.load(f)
    bf_results = {dim: {fid: bf_all[dim][fid][0] for fid in fids} for dim in dimensions}

    ga_corr = np.zeros((11, 11))
    bf_corr = np.zeros((11, 11))

    for dim in dimensions:
        for fid in fids:
            ga = ga_results[dim][fid]
            bf = bf_results[dim][fid]

            for i in range(11):
                for j in range(i, 11):
                    ga_corr[i, j] += 1 if ga[i] * ga[j] != 0 else 0
                    bf_corr[i, j] += 1 if bf[i] * bf[j] != 0 else 0

    print(ga_corr)
    print(bf_corr)

def printComparisonGivenInBF(given=None):

    # Add/remove choices as you wish
    if given is None:
        given = default_ESs

    for dim in dimensions:
        for fid in functions:
            print("Results for F{} in {}dim:".format(fid, dim))
            results = findGivenInRankedBF(dim, fid, given)
            for ES in results:
                print("Rank: {0:>4}\t{1:>33}\t{2}".format(ES.rank + 1, intToRepr(ES.ES), ES.fitness))
            print()

def printRelativeComparisonBestAndGivenBF(given=None):

    # Add/remove choices as you wish
    if given is None:
        given = default_ESs

    os.chdir(brute_location)
    with open('brute_results.dat') as f:
        brute_results = cPickle.load(f)

    all_relatives = []
    relatives = {i: [] for i in range(len(default_ESs))}
    counts = [0] * len(default_ESs)
    for dim in dimensions:
        for fid in functions:
            print("Results for F{} in {}dim:".format(fid, dim))
            results = findGivenInRankedBF(dim, fid, given)
            ranks = []
            for i, ES in enumerate(results):
                if ES.fitness.ERT is not None:
                    relative_fitness = ES.fitness.ERT / brute_results[dim][fid].fitness.ERT
                    relatives[i].append(relative_fitness)
                elif brute_results[dim][fid].fitness.ERT is not None and brute_results[dim][fid].fitness.FCE > ES.fitness.FCE:
                    relative_fitness = "Unclear???"
                else:
                    relative_fitness = ES.fitness.FCE / brute_results[dim][fid].fitness.FCE
                all_relatives.append(relative_fitness)
                ranks.append(ES.rank)
                print("Rank: {0:>4}\t{1:>33}\t{2}".format(ES.rank + 1, intToRepr(ES.ES), relative_fitness))
            for i, rank in enumerate(ranks):
                if rank == min(ranks):
                    counts[i] += 1
            print()

    print(all_relatives)
    print(relatives)
    print(counts)

def printRelativeComparisonBestGAAndGivenBF(given=None):

    # Add/remove choices as you wish
    if given is None:
        given = default_ESs

    os.chdir(ga_location)
    with open('ga_results.dat') as f:
        ga_results = cPickle.load(f)

    all_relatives = []
    relatives = {i: [] for i in range(len(default_ESs))}
    counts = [0] * len(default_ESs)
    for dim in dimensions:
        for fid in functions:
            print("Results for F{} in {}dim:".format(fid, dim))
            results = findGivenInRankedBF(dim, fid, given)
            ranks = []
            for i, ES in enumerate(results):
                if ES.fitness.ERT is not None:
                    relative_fitness = ES.fitness.ERT / ga_results[dim][fid].fitness.ERT
                    relatives[i].append(relative_fitness)
                elif ga_results[dim][fid].fitness.ERT is not None and ga_results[dim][fid].fitness.FCE > ES.fitness.FCE:
                    relative_fitness = "Unclear???"
                else:
                    relative_fitness = ES.fitness.FCE / ga_results[dim][fid].fitness.FCE
                all_relatives.append(relative_fitness)
                ranks.append(ES.rank)
                print("Rank: {0:>4}\t{1:>33}\t{2}".format(ES.rank + 1, intToRepr(ES.ES), relative_fitness))
            for i, rank in enumerate(ranks):
                if rank == min(ranks):
                    counts[i] += 1
            print()

    print(all_relatives)
    print(relatives)
    print(counts)

def printComparisonTable(given=None):
    # Add/remove choices as you wish
    if given is None:
        given = default_ESs

    os.chdir(ga_location)
    with open('ga_results.dat') as f:
        ga_results = cPickle.load(f)

    print('\\hline')
    print('F-ID & N & Brute Force & ERT & FCE & GA & ERT & FCE & Relative \\\\')
    print('\\hline')
    print('\\hline')

    for fid in functions:
        for dim in dimensions:
            results = findGivenInRankedBF(dim, fid, given)
            results.sort(key=lambda a: a.rank)

            bf_string = reprToString(intToRepr(results[0].ES))
            bf_ERT = "{:>9.6g}".format(results[0].fitness.ERT) if results[0].fitness.ERT is not None else "      N/A"
            bf_FCE = results[0].fitness.FCE

            ga_string = reprToString(ga_results[dim][fid].ES[:11])
            ga_ERT = "{:>9.6g}".format(ga_results[dim][fid].fitness.ERT) if ga_results[dim][fid].fitness.ERT is not None else "      N/A"
            ga_FCE = ga_results[dim][fid].fitness.FCE

            if results[0].fitness.ERT is not None:
                relative_fitness = results[0].fitness.ERT / ga_results[dim][fid].fitness.ERT
            else:
                relative_fitness = results[0].fitness.FCE / ga_results[dim][fid].fitness.FCE

            print('F{0:<2} & {1:>2} & {2} & {3} & {4:>9.4g} & {5} & {6} & {7:>9.4g} & {8:>8.3g} \\\\'.format(
                fid, dim, bf_string, bf_ERT, bf_FCE, ga_string, ga_ERT, ga_FCE, relative_fitness
            ))
        print('\\hline')

def datToArff(input, output, dim):

    arff_header = """
% 1. Title: Modular CMA-ES Framework - Fitness values per structure
%
% 2. Sources:
%      (a) Creator: S.J. van Rijn (s.j.van.rijn@liacs.leidenuniv.nl)
%      (b) Donor: Sander van Rijn
%      (c) Date: January, 2017
%
% 3. Remark:
%      The 'class' value is the pre-computed q-score.
%      This q-score is defined as follows:
%
%      q = ERT/max_ERT       if ERT exists
%        = 1 + FCE/max_FCE   else
%
%      where for this dataset, max_ERT = {ERT} and max_FCE = {FCE}
%
@RELATION ESs

@ATTRIBUTE active       NUMERIC
@ATTRIBUTE elitist      NUMERIC
@ATTRIBUTE mirrored     NUMERIC
@ATTRIBUTE orthogonal   NUMERIC
@ATTRIBUTE sequential   NUMERIC
@ATTRIBUTE threshold    NUMERIC
@ATTRIBUTE tpa          NUMERIC
@ATTRIBUTE pairwise     NUMERIC
@ATTRIBUTE weights      NUMERIC
@ATTRIBUTE base-sampler NUMERIC
@ATTRIBUTE ipop         NUMERIC
@ATTRIBUTE class        NUMERIC

@DATA
"""

    fitnesses = tdatFileToFitnesses(input)

    ERTs, FCEs = zip(*[(case.fitness.ERT, case.fitness.FCE) for case in fitnesses])
    ERTs = [ert for ert in ERTs if ert is not None]

    with open(output, 'w') as f:

        max_ERT = 32 * 1000 * dim
        max_FCE = max(FCEs)

        f.write(arff_header.format(ERT=max_ERT, FCE=max_FCE))

        for single_result in fitnesses:

            # Normalize ERT (assumed linear) to [0, 1]
            if single_result.fitness.ERT is not None\
                             and max_ERT is not None:
                fitness = single_result.fitness.ERT / max_ERT
            # Normalize FCE (assumed log-linear) to [1, 2] if an ERT is not available
            else:
                fitness = 1 + ((np.log(single_result.fitness.FCE) + 8) / (np.log(max_FCE) + 8))

            line = "{},{},{},{},{},{},{},{},{},{},{},{}\n".format(
                single_result.ES[0],
                single_result.ES[1],
                single_result.ES[2],
                single_result.ES[3],
                single_result.ES[4],
                single_result.ES[5],
                single_result.ES[6],
                single_result.ES[7],
                single_result.ES[8],
                single_result.ES[9],
                single_result.ES[10],
                fitness
            )
            f.write(line)

def createArffFiles():

    os.chdir("/media/Data/Research Data/Evolving ES data")

    print(os.getcwd())

    for fid in functions:
        for dim in dimensions:
            tdat_name = raw_bfname.format(dim, fid)
            arff_name = "arff/bruteforce_{}_f{}.arff".format(dim, fid)
            datToArff(tdat_name, arff_name, dim)

if __name__ == '__main__':

    ### GA STUFF ###
    # storeBestFromGA()
    storeResults()
    # printResults()

    # createGARunPlots()
    printGATable()
    # printGAcount()
    # storeRepresentation()
    # GAtimeSpent()

    # os.chdir(ga_location)
    # with open('ES_per_experiment.json') as infile:
    #     x = json.load(infile)
    # pprint.pprint(x)

    ### Brute Force STUFF ###

    # checkFileSizesBF()
    # storeBestFromBF()
    # printBFFitDistances()

    # findGAInRankedBF()
    # printGAInRankedBF()
    # printDoubleTable()
    # printDoubleCount()
    # correlationMatrix()

    # for i, subgroup in enumerate(subgroups):
    #     print(i)
    #     printDoubleCount(fids=subgroup)
    #     correlationMatrix(fids=subgroup)

    # for dim in dimensions:
    #     print(dim)
    #     printDoubleCount(dims=[dim])

    # printComparisonGivenInBF()
    # printBestFromRankedBF()

    # printRelativeComparisonBestAndGivenBF()
    # printRelativeComparisonBestGAAndGivenBF()

    #printComparisonTable()

    createArffFiles()

    pass


import os
import music21 as m21
import torch
import torch.nn as nn
from collections import defaultdict
from torch.nn.utils.rnn import pad_sequence
import random
import numpy as np

def tokenize_score(score, time_step=0.25):
    tokens = []
    for part in score.parts:
        part_stream = part.flatten().notesAndRests
        prev_event = None
        symbol = -1  # Default value for 'symbol' variable

        for event in part_stream:
            if isinstance(event, m21.note.Note):
                symbol = event.pitch.midi
            elif isinstance(event, m21.note.Rest):
                symbol = -1  # Use -1 to represent rests

            steps = int(event.duration.quarterLength / time_step)
            tokens.extend([symbol] + [-1] * (steps - 1))

    return tokens

def load_tokenized_songs(dataset_path, segment_length=100, step_size=10, max_neg_ones=30):
    songs = []
    for path, _, files in os.walk(dataset_path):
        for file in files:
            song = m21.converter.parse(os.path.join(path, file))
            tokens = tokenize_score(song)
            num_segments = (len(tokens) - segment_length) // step_size + 1
            for i in range(num_segments):
                segment = tokens[i * step_size : i * step_size + segment_length]
                if segment.count(-1) <= max_neg_ones:
                    songs.append(segment)
    return songs

def process_files(folder_path):
    element_dict = defaultdict(int)

    files = os.listdir(folder_path)
    files.sort()

    for file in files:
        file_path = os.path.join(folder_path, file)

        with open(file_path, 'r') as f:
            content = f.read().split()

        for element in content:
            element_dict[element] += 1

    element_mapping = {element: i for i, element in enumerate(element_dict.keys())}
    reversed_mapping = {v: k for k, v in element_mapping.items()}  # Reverse the mapping

    return reversed_mapping

def one_hot_encoding(folder_path, element_mapping, desired_length):
    encoded_tensors = []  # List to store encoded tensors

    files = os.listdir(folder_path)
    files.sort()

    all_elements = set(element_mapping.values())

    for file in files:
        file_path = os.path.join(folder_path, file)

        with open(file_path, 'r') as f:
            content = f.read().split()

        encoded_content = [element_mapping.get(element, 0) for element in content]
        encoded_content = torch.tensor(encoded_content)  # Convert to PyTorch tensor

        # Pad or truncate the tensor to the desired length
        if len(encoded_content) < desired_length:
            padded_content = nn.functional.pad(encoded_content, (0, desired_length - len(encoded_content)))
        else:
            padded_content = encoded_content[:desired_length]

        encoded_tensors.append(padded_content)

    # Pad sequences to the same length
    padded_sequences = pad_sequence(encoded_tensors, batch_first=True)

    encoded_arrays = []
    for tensor, file in zip(padded_sequences, files):

        # Convert tensor to numpy array
        onehot_encoded = np.zeros((tensor.size(0), len(all_elements)))
        for i, element in enumerate(tensor):
            if element.item() in element_mapping.values():
                onehot_encoded[i, element.item()] = 1

        encoded_arrays.append((onehot_encoded, file))

    return encoded_arrays


import bs4
from mitmproxy import http
import htmlmin
from utils import Utils

class Fandom:
    async def response(self, flow: http.HTTPFlow) -> None:
        if not flow.request.host.endswith("fandom.com"): return # host end

        if flow.response.status_code != 200: return  # process HTTP 200 only
        if len(flow.response.content) == 0: return  # skip empty responses
        if not Utils.is_html(flow): return # proccess html only

        soup = bs4.BeautifulSoup(flow.response.content, 'lxml')
        container = soup.select_one(".top-ads-container")
        container.decompose()
        flow.response.content = htmlmin.minify(soup.prettify(), remove_empty_space=True).encode(encoding='utf-8')

        if flow.request.host.startswith("genshin-impact"):
            if flow.request.path == "/wiki/Event":
                await Utils.inject(
                    flow,
                    {
                        "scripts": [
                            {"path": Utils.local_injector_url("fandom.com/genshin-impact/event_list.js")},
                        ],
                    }
                )
            if flow.request.path == "/wiki/World_Quest/List":
                await Utils.inject(
                    flow,
                    {
                        "scripts": [
                            {"path": Utils.local_injector_url("fandom.com/genshin-impact/world_quests_list.js")},
                        ],
                    }
                )
            if flow.request.path == "/wiki/Hidden_Exploration_Objective":
                await Utils.inject(
                    flow,
                    {
                        "scripts": [
                            {"path": Utils.local_injector_url("fandom.com/genshin-impact/hidden_exploration_objective.js")},
                        ],
                    }
                )


# module yg disediakan python
import numpy as np
import cv2
import math

# module yg dibuat sendiri
import SubProcess.BitOperation as bo
import SteganographyMethod.CharLength as cl

# fungsi mendapatkan nilai dari tabel kuantisasi
def qTable(val):
    if val in range(0,8):
        return 0, 3
    elif val in range(8,16):
        return 8, 3
    elif val in range(16,32):
        return 16, 3
    elif val in range(32,64):
        return 32, 3
    elif val in range(64,128):
        return 64, 3
    elif val in range(128,256):
        return 128, 3

# fungsi encode pesan
def encode(img, message):
    # konversi gambar ke grayscale
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # proses merubah pesan string menjadi bit
    bitMessage = bo.word2bit(message)

    # mendapatkan panjang pesan setelah dirubah menjadi bit
    bitLenght = len(bitMessage)
    index = 0

    # proses mendapatkan jumlah baris dan kolom
    rows, cols = img.shape[:2]
    if rows % 2 == 1:rows += 1
    if cols % 2 == 1:cols += 1
    img = cv2.resize(img, (cols, rows))

    # inisialisasi image baru tempat pesan akan disimpan
    imgResult = np.zeros((rows, cols,1),np.uint8)*255
    lastIteration = True
    isFinish = True

    # proses menyisipkan pesan ke dalam gambar dengan aturan pixel value differencing
    # karena citra grayscale, setiap pixel dapat menampung minimal 1 bit pesan. Setiap bit disisipkan dalam setiap pixel
    for i in range(rows):
        j = 0
        while j < cols:
            color = int(img[i,j])
            if lastIteration:
                gi, gi1 = int(img[i,j]), int(img[i,j+1])
                diff = gi1 - gi
                ik, n = qTable(abs(diff))
                restBit = len(bitMessage)
                if restBit <= n:
                    bitMessage.extend([0 for i in range(n - restBit)])
                    lastIteration = False
                bit = bitMessage[:n]
                del bitMessage[:n]
                b = bo.bit2int(bit)

                diffA = 0
                if diff >= 0:
                    diffA = ik + b
                else:
                    diffA = -1 * (ik + b)

                m = abs(diffA - abs(diff))
                if m % 2 == 0:
                    gi -= int(m/2)
                    gi1 += int(m/2)
                    while abs(gi - gi1) != b:
                        if gi < gi1:
                            gi += 1
                        else:
                            gi1 += 1
                else:
                    gi -= math.ceil(m/2)
                    gi1 += math.floor(m/2)
                    while abs(gi - gi1) != b:
                        if gi < gi1:
                            gi += 1
                        else:
                            gi1 += 1

                if gi < 0:
                    gi1 -= gi
                    gi = 0

                if gi1 > 255:
                    gi -= gi1 - 255
                    gi1 = 255

                imgResult[i,j] = gi
                imgResult[i,j+1] = gi1
                j += 2
            else:
                imgResult[i,j] = color
                j += 1

    # kombinasi nilai val1, val2, val3 akan menyimpan nilai infromasi panjang pesan.
    # Nilai ini dibutuhkan ketika proses decode
    val1, val2, val3, val4 = cl.setCharLength(len(message))

    # ketiga nilai tersebut akan disimpan pada 3 pixel terakhir
    imgResult[rows-1, cols-1] = val1
    imgResult[rows-1, cols-2] = val2
    imgResult[rows-1, cols-3] = val3
    imgResult[rows-1, cols-4] = val4

    return imgResult

def decode(img):
    # konversi gambar ke grayscale
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # proses mendapatkan resolusi gambar
    rows, cols = img.shape[:2]

    # proses mendapatkan nilai panjang pesan. Sehingga proses extraksi akan berhenti ketika semua pesan sudah diextract
    charLength = cl.getCharLenth(img[rows-1, cols-1], img[rows-1, cols-2], img[rows-1, cols-3], img[rows-1, cols-4])

    # panjang karakter dikalikan dengan 8 karena 1 karakter = 8 bit integer
    charLength = charLength * 8
    index = 0
    bit = []

    # proses extraksi pesan dengan cara menelusuri setiap pixel pada gambar
    # proses menelusuri akan berhenti ketika semua pesan sudah diextract
    for i in range(rows):
        j = 0
        while j < cols:
            if index < charLength:
                gi, gi1 = int(img[i,j]), int(img[i,j+1])
                diff = abs(gi - gi1)
                ik, n = qTable(diff)
                b = bo.int2bit(abs(diff - ik))
                fixBit = [0 for i in range(n - len(b))]
                fixBit.extend(b)
                bit.extend(fixBit)
                index += n
                img[i,j] -= diff
                if img[i,j] < 0:
                    img[i,j] = 0
            else:
                break
            j += 2

    cv2.imwrite('../img/pvd-gray/after-decode.png', img)
    return bo.bit2word(bit)


# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import psycopg2

class LitpricehunterPipeline:
    def open_spider(self, spider):
        # Установите соединение с базой данных PostgreSQL
        self.conn = psycopg2.connect(
            database="",  # Замените на имя вашей базы данных
            user="",          # Замените на имя пользователя базы данных
            password="",  # Замените на пароль пользователя базы данных
            host="",          # Замените на хост базы данных (обычно "localhost")
            port="",         # Замените на порт базы данных (обычно 5432)
            client_encoding="UTF-8"
        )
        self.cursor = self.conn.cursor()

    def close_spider(self, spider):
        # Закрываем соединение с базой данных PostgreSQL и сохраните изменения
        self.conn.commit()
        self.conn.close()

    def process_item(self, item, spider):
        # Вставляем данные в таблицу "books" в базе данных PostgreSQL

        # Запрос на вставку данных в таблицу "books" с использованием параметров
        insert_query = """
            INSERT INTO books (title, author, price, link)
            VALUES (%s, %s, %s, %s)
        """

        # Выполняем запрос на вставку данных, передавая значения из элемента item
        self.cursor.execute(insert_query, (item['title'], item['author'], item['price'], item['link']))

        # Возвращаем элемент item, чтобы он мог быть обработан дальше
        return item


#!/usr/bin/env python
#-*- encoding: utf8 -*-

from __future__ import print_function
from collections import namedtuple
from pyelftools.elftools.elf.elffile import ELFFile as ELFFile_
from pyelftools.elftools.elf.sections import Section
from pyelftools.elftools.common.py3compat import bytes2str

Func = namedtuple("Func", ["name", "mangled_name", "offset", "size"])

if __package__ is None:
    from os import sys, path
    sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))

class ELFFileError(Exception):
    def __init__(self, what):
        Exception.__init__(self, what)

class ELFFile(ELFFile_):
    def __init__(self, filepath):
        self._fd = open(filepath, 'rb')
        ELFFile_.__init__(self, self._fd)
        self._symbol_interval_tree = self._build_symbol_tree()

    def __del__(self):
        self._fd.close()

    def get_offset_for_virtual_address(self, vaddr):
        correct_segment = None
        for segment in self.iter_segments():
            if (segment['p_vaddr'] >= vaddr and
                vaddr < segment['p_vaddr'] + segment['p_filesz']):
                correct_segment = segment
                break

        if correct_segment is not None:
            correct_section = None
            for section in self.iter_sections():
                if correct_segment.section_in_segment(section):
                    correct_section = section
                    break
            if correct_section is not None:
                offset_in_segment = vaddr - correct_segment['p_vaddr']
                return correct_section['sh_offset'] + offset_in_segment
            else:
                return None
        else:
            return None

    def get_virtual_address_for_offset(self, offset):
        section = self.get_section_by_offset(offset)
        offset_from_section_start = offset - section['sh_offset']
        section_vaddr = self.get_section_virtual_address(section)
        return section_vaddr + offset_from_section_start

    def get_section_by_offset(self, offset):
        for section in self.iter_sections():
            if offset >= section['sh_offset'] and offset < section['sh_offset'] + section['sh_size']:
                return section
        return None

    def get_segment_by_virtual_address(self, vaddr):
        for segment in self.iter_segments():
            if segment['p_vaddr'] >= vaddr and vaddr < segment['p_vaddr'] + segment['p_filesz']:
                return segment
        return None

    def get_section_by_virtual_address(self, vaddr):
        segment = self.get_segment_by_virtual_address(vaddr)
        for section in self.iter_section():
            if segment.section_in_segment(section):
                return section
        return None

    def get_section_virtual_address(self, section):
        assert isinstance(section, Section)
        matching_segments = []
        for segment in self.iter_segments():
            if segment.section_in_segment(section):
                matching_segments.append(segment)
        return max(matching_segments, key=lambda s: s['p_filesz'])

    def get_symbol_by_name(self, name):
        for symbol in self.iter_symbols():
            if symbol.name == name:
                return symbol
        # if we reached this point, then there is no such symbol in the file
        raise ELFFileError("The file has no %s symbol" % name)

    def get_function_by_mangled_name(self, mangled_name):
        for func in self.iter_functions():
            if func.mangled_name == mangled_name:
                return func
        raise ELFFileError("There is no function with mangled name %s" % mangled_name)

    def get_function_by_name(self, name):
        for func in self.iter_functions():
            if func.name == name:
                return func
        raise ELFFileError("There is no %s function" % name)

    def iter_symbols(self):
        sym_name = b".symtab"
        symtab = self.get_section_by_name(sym_name)
        if symtab is None:
            raise ELFFileError('The file has no %s section' % bytes2str(sym_name))
        return symtab.iter_symbols()

    def iter_functions(self):
        symtab = self._get_symbol_table()
        return self._iter_func(symtab.iter_symbols())

    def get_symbol_text(self, name):
        if name not in self.get_symbol_names():
            raise ELFFileError('The file has no % symbol' % name)
        text_section_offset = self._get_text_offset()
        symbol = self.get_symbol_by_name(name)
        print(symbol.name)

        # rewind the file to be at the symbol position
        symbol_size = symbol.entry["st_size"]
        symbol_offset = symbol.entry["st_value"]

        # remember the current position in order to go back afterwards
        cur = self._fd.tell()
        self._fd.seek(symbol_offset-text_section_offset)
        text = self._fd.read(symbol_size)
        self._fd.seek(cur)
        return text

    def get_function_names(self):
    # print('Processing file:', filename)
        sym_name = b'.symtab'
        symtab = self.get_section_by_name(sym_name)
        if symtab is None:
            raise ELFFileError('The file has no %s section' % bytes2str(sym_name))
        return [func.name for func in self._iter_func(symtab.iter_symbols())]

    def get_symbol_names(self):
        return [symbol.name for symbol in self.iter_symbols()]

    def get_section_names(self):
        return [symbol.name for symbol in self.iter_sections()]

    def get_text(self):
        text = self.get_section_by_name('.text')

        if text is None:
            raise ELFFileError("ELF file has no .text section")
        return text.data()

    def get_section_by_offset(self, offset):
        for section in  self.iter_sections():
            if self._offset_inside_section(offset, section):
                return section
        raise ELFFileError("There is no section that contains given offset: %d" % offset)

    def get_symbol_by_offset(self, offset):
        """
        Figures out name of the symbol for the given offset.

        offset -- a number expressing the offset with substracted base

        Returns None or a string. None is returned when there is no corresponding symbol
        (e.g. the binary is stripped or the offset is malformed)
        Otherwise returns demangled symbol name
        """
        symbol = self._symbol_interval_tree.search(offset) # returns a set
        if not symbol: #an empty set
            return None
        else:
            #assert len(symbol) == 1
            return list(symbol)[0].data # sets are not indexed, must be converted to a list

    def get_text_by_offset(self, offset):
        pass

### private functions ###

    def _get_text_offset(self):
        text_section = self.get_section_by_name('.text')
        for segment in self.iter_segments():
            if segment.section_in_segment(text_section):
                return segment['p_paddr']
        raise ELFFileError("Section .text is not present in the ELF file")

    def _get_symbol_table(self):
        sym_name = b'.symtab'
        symtab = self.get_section_by_name(sym_name)
        return symtab

    def _iter_func(self, symbols_iter=None):
        from ..demangle import demangle
        if symbols_iter is None:
            symtab = self._get_symbol_table()
            symbols_iter = symtab.iter_symbols()

        for sym in symbols_iter:
            if sym.entry['st_info']['type'] == 'STT_FUNC':
                demangled_name = demangle.cplus_demangle(sym.name, 1)
                if demangled_name is not None:
                    name = demangled_name
                else:
                    name = sym.name
                yield Func(name=name,
                           mangled_name=sym.name,
                           offset=sym.entry['st_value'],
                           size=sym.entry['st_size'])
        raise StopIteration()

    def _offset_inside_section(self, offset, section):
        sh = section.header
        return (offset >= sh['sh_offset']) and (offset < sh['sh_offset']+sh['sh_size'])

    def _build_symbol_tree(self):
        from intervaltree import IntervalTree, Interval
        base = self._get_binary_base()
        intervals = [Interval(func.offset-base, func.offset+func.size-base, func.name) for func in self._iter_func() if func.size != 0]
        return IntervalTree(intervals)

    def _get_binary_base(self):
        for segment in self.iter_segments():
            if segment['p_type'] == "PT_LOAD" and segment['p_offset'] == 0:
                return segment['p_vaddr']
        raise ELFFileError("Can't find base for the .text segment")

  #  def _offset_inside_segment(self, offset, segment):
  #      sh = segment.header
  #      return (offset >= sh['p_paddr']) and (offset < sh['p_paddr']+sh['p_filesz'])

if __name__ == "__main__":
    elf = ELFFile("/afs/cern.ch/user/p/paszoste/cp/simple-binary/a.out")
    print(elf.get_symbol_by_offset(int('731', 16)))
    print(elf.get_symbol_by_offset(int('982', 16)))
    print(elf.get_symbol_by_offset(int('9cd', 16)))
    print(elf.get_symbol_by_offset(int('9f5', 16)))
    print(elf.get_symbol_by_offset(int('a42', 16)))
    # for symbol in elf.iter_symbols():
    #     print(symbol.name, symbol.entry["st_size"], symbol.entry["st_value"])
    textsec = elf.get_section_by_name('.text')
    # print("SECTION")
    # for sec in elf.iter_sections():
    #     print(sec.header)
    #     if sec['sh_type'] == 'SHT_STRTAB':
    #         print(sec.get_string(49))
    # print("SEGMENT")
    # for seg in elf.iter_segments():
    #     print(seg.section_in_segment(textsec), seg.header)
    # print(elf._get_text_offset())
    text = elf.get_symbol_text("main")
    main_file = open("main_text", "wb")
    for segment in elf.iter_sections():
        print(segment.header)
    main_file.write(text)
    print(len(text))
    import xed
    inst_list = xed.disassemble(1, text, len(text))
    print(len(inst_list))
    for inst in inst_list:
        print(inst)
        print(xed.xed_decoded_inst_dump_intel_format(inst, 1024, 0))

{'demangle': 'demangle.demangle', 'IntervalTree': 'intervaltree.IntervalTree', 'Interval': 'intervaltree.Interval'}