import socket
import threading
import config

class Client:
    def __init__(self, name):
        self.name = name
        self.channel = "main"

class ServerApp:
    def __init__(self):
        self.host = "127.0.0.1"
        self.port = config.PORT
        self.clients = {}
        self.channels = ["main"]
        self.isRunning = False

    def start(self):
        if (self.isRunning):
            return

        try:
            self.isRunning = True
            self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.socket.bind((self.host, self.port))
            self.socket.listen()
            print(f"[*] Listening on {self.host}:{self.port}")
        except socket.error as e:
            print(f"[!] Unable to start server: {str(e)}")

    # append a new line character and the encode message
    # before sending it to a client
    def apply_message_preprocessing(self, message):
        return (message + "\n").encode(config.ENCODING)

    def send_message_to_channel(self, message, channel):
        message = self.apply_message_preprocessing(message)
        for k, v in self.clients.items():
            if (v.channel == channel):
                k.send(message)

    def broadcast_message(self, message):
        message = self.apply_message_preprocessing(message)
        for client in self.clients:
            client.send(message)

    def send_message_to_client(self, client, message):
        message = self.apply_message_preprocessing(message)
        client.send(message)

    def handle_direct_message(self, message, recipientName, sender, senderConnection):
        isRecipientFound = False
        for k, v in self.clients.items():
            if (v.name == recipientName):
                isRecipientFound = True
                self.send_message_to_client(senderConnection, f"You -> {recipientName}: {message}")
                self.send_message_to_client(k, f"{sender.name} -> {recipientName}: {message}")

        if (not isRecipientFound):
            self.send_message_to_client(senderConnection, "[e] Recipient not found")

    def add_client_chat_formatting(self, client, message):
        return f"{client.name}: {message}"

    def new_client_thread(self, client, address):
        threading.Thread(target=self.client_handler, args=(client, address)).start()

    def does_channel_exist(self, value):
        for channel in self.channels:
            if (channel == value):
                return True
        return False

    def client_handler(self, connection, address):
        # ask nickname
        connection.send(config.CHANGE_NICKNAME_CMD.encode(config.ENCODING))
        nickname = connection.recv(config.BUFFER_SIZE).decode(config.ENCODING)

        client = Client(nickname)
        client.address = address
        self.clients[connection] = client
        self.send_message_to_client(
            connection, f"Welcome to the server's #{client.channel} channel")
        self.broadcast_message(f"[+] {client.name} has joined the server.")
        self.send_message_to_channel(
            f"[+] {client.name} has joined the channel.", client.channel)

        while self.isRunning:
            try:
                data = connection.recv(
                    config.BUFFER_SIZE).decode(config.ENCODING)

                if (config.CHANGE_NICKNAME_CMD in data):
                    parts = data.split(";")
                    if (len(parts) == 2):
                        oldName = client.name
                        client.name = parts[1]
                        self.send_message_to_client(connection, f"[i] Name changed from '{oldName}' to '{client.name}'")
                    else:
                        self.send_message_to_client(connection, "[e] Invalid command")
                elif (config.CREATE_CHANNEL_CMD in data):
                    parts = data.split(";")
                    if (len(parts) == 2):
                        channelName = parts[1]
                        if (self.does_channel_exist(channelName)):
                            self.send_message_to_client(connection, "[e] Channel already exists")
                        else:
                            self.channels.append(channelName)
                            self.send_message_to_client(connection, f"[i] Created channel '{channelName}'")
                    else:
                        self.send_message_to_client(connection, "[e] Invalid command")
                elif (config.JOIN_CHANNEL_CMD in data):
                    parts = data.split(";")
                    if (len(parts) == 2):
                        channelName = parts[1]
                        if (client.channel == channelName):
                            self.send_message_to_client(connection, "[e] Already in this channel")
                        elif (self.does_channel_exist(channelName)):
                            self.send_message_to_channel(
                                f"[-] {client.name} has left the channel.", client.channel)
                            client.channel = channelName
                            self.send_message_to_channel(
                                f"[+] {client.name} has joined the channel.", client.channel)
                        else:
                            self.send_message_to_client(connection, "[e] Channel does not exist")
                    else:
                        self.send_message_to_client(connection, "[e] Invalid command")
                elif (config.DIRECT_MESSAGE_CMD in data):
                    parts = data.split(";")
                    if (len(parts) == 3):
                        recipientName = parts[1]
                        message = parts[2]
                        self.handle_direct_message(
                            message, recipientName, client, connection)
                    else:
                        self.send_message_to_client(connection, "[e] Invalid command")
                else:
                    self.send_message_to_channel(
                        self.add_client_chat_formatting(client, data), client.channel)
            except socket.error as e:
                print(f"[-] {client.address} disconnected.")
                del self.clients[connection]
                self.send_message_to_channel(
                    f"[-] {client.name} has left the channel.", client.channel)
                self.broadcast_message(
                    f"[-] {client.name} has left the server.")
                connection.close()
                break

    def stop(self):
        self.isRunning = False
        self.socket.close()

if __name__ == "__main__":
    app = ServerApp()
    app.start()

    try:
        while True:
            client, address = app.socket.accept()
            print(f"[+] {address} connected.")
            app.new_client_thread(client, address)
    except KeyboardInterrupt:
        app.stop()
        print("[i] Exiting...")


import sqlite3

class Personne:
    def __init__(self):
        self.nom = "a",
        self.prenom = "b"
        self.age = "0"

    def getConnection(self):
        try:

            cnx = sqlite3.connect('presonne.db')
            return cnx

        except Exception as e:
            return e

    def createTable(self, cnx):

        # cnx.execute("DROP TABLE personne")
        cnx.execute("CREATE TABLE personne(""id INT AUTO_INCREMENT PRIMARY KEY NOT NULL  ,""nom VARCHAR(50),""age INT)")

        cnx.close()

        def insertData(self,cnx,pers):
            cnx.execute("INSERT INTO personne VALUES(""{},{}}))".format(['nom'],pers['age']))

            cnx.commit()
            cnx.close()

pers1 = Personne()
cnx = pers1.getConnection()
#creat table personn1

nom = input("Entrer un nom:")
age = int(input("Entrer age:"))
#pers1.createTable(cnx)
persona ={"nom": nom,"age":age}


import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk

APPIND_SUPPORT = 1
try:
    gi.require_version('AppIndicator3', '0.1')
    from gi.repository import AppIndicator3
except ValueError:
    APPIND_SUPPORT = 0

class SubMenu(Gtk.Menu):
    def __init__(self, *args, **kwargs):
        Gtk.Menu.__init__(self)

    def add_command(self, label="", command=None):
        item = Gtk.MenuItem(label=label)
        self.append(item)
        if command is not None:
            item.connect("activate", lambda *args: command())
        item.show()

    def add_cascade(self, label="", menu=None):
        item = Gtk.MenuItem(label=label)
        self.append(item)
        if menu is not None:
            item.set_submenu(menu)
        item.show()

    def add_separator(self):
        sep = Gtk.SeparatorMenuItem()
        self.append(sep)
        sep.show()

    def delete(self, item1, item2=None):
        index1 = self.index(item1)
        if item2 is None:
            self.remove(self.get_children()[index1])
        else:
            index2 = self.index(item2)
            c = self.get_children()
            for i in range(index1, index2):
                self.remove(c[i])

    def index(self, index):
        if isinstance(index, int):
            return index
        elif index == "end":
            return len(self.get_children())
        else:
            try:
                i = [i.get_label() for i in self.get_children()].index(index)
            except ValueError:
                raise ValueError("%r not in menu" % index)
            return i

    def get_item_label(self, item):
        return self.get_children()[self.index(item)].get_label()

    def set_item_label(self, item, label):
        self.get_children()[self.index(item)].set_label(label)

    def get_item_menu(self, item):
        return self.get_children()[self.index(item)].get_submenu()

    def set_item_menu(self, item, menu):
        self.get_children()[self.index(item)].set_submenu(menu)

    def disable_item(self, item):
        self.get_children()[self.index(item)].set_sensitive(False)

    def enable_item(self, item):
        self.get_children()[self.index(item)].set_sensitive(True)

class TrayIcon:
    """Gtk system tray icon."""
    def __init__(self, icon, fallback_icon_path, appid="TrayIcon", **kwargs):
        self.menu = SubMenu()

        icon_exists = Gtk.IconTheme.get_default().has_icon(icon)

        if APPIND_SUPPORT == 1:
            if icon_exists:
                self.tray_icon = AppIndicator3.Indicator.new(appid, icon,
                                                             AppIndicator3.IndicatorCategory.APPLICATION_STATUS)
            else:
                self.tray_icon = AppIndicator3.Indicator.new(appid,
                                                             fallback_icon_path,
                                                             AppIndicator3.IndicatorCategory.APPLICATION_STATUS)
            self.tray_icon.set_status(AppIndicator3.IndicatorStatus.ACTIVE)
            self.tray_icon.set_menu(self.menu)
            self.change_icon = self._change_icon_appind
        else:
            if icon_exists:
                self.tray_icon = Gtk.StatusIcon.new_from_icon_name(icon)
            else:
                self.tray_icon = Gtk.StatusIcon.new_from_file(fallback_icon_path)
            self.tray_icon.connect('popup-menu', self._on_popup_menu)
            self.change_icon = self._change_icon_fallback

    def _on_popup_menu(self, icon, button, time):
        self.menu.popup(None, None, Gtk.StatusIcon.position_menu, icon, button, time)

    def _change_icon_appind(self, icon, desc):
        self.tray_icon.set_icon_full(icon, desc)

    def _change_icon_fallback(self, icon, desc):
        self.tray_icon.set_from_file(icon)

    def loop(self, tk_window):
        """Update Gtk GUI inside tkinter mainloop."""
        while Gtk.events_pending():
            Gtk.main_iteration()
        tk_window.loop_id = tk_window.after(10, self.loop, tk_window)

    def bind_left_click(self, command):
        if not APPIND_SUPPORT:
            self.tray_icon.connect('activate', lambda *args: command())


import os
import datetime
from tradetools.FoundationHelper import get_jingzhi

from flask_sqlalchemy import SQLAlchemy

db = SQLAlchemy()

WORKER_STOP = 0
WORKER_RUN = 1

CRASH_UNCHECKED = 0
CRASH_NEGATIVE = 1
CRASH_POSITIVE = 2

class Worker(db.Model):
    id = db.Column(db.String(36), primary_key=True)
    ip = db.Column(db.String(15))
    bind_port = db.Column(db.Integer)
    proc_count = db.Column(db.Integer)
    case_count = db.Column(db.Integer, default=0)
    log_path = db.Column(db.String(4096))
    status = db.Column(db.Integer)
    started_at = db.Column(db.DateTime)
    resource = db.relationship('Resource', backref='host', uselist=False)
    crashes = db.relationship('Crash', backref='host', lazy='dynamic')

    def __init__(self, id=None, ip=None, bind_port=None, proc_count=None, log_path=None, status=WORKER_RUN):
        self.id = id
        self.ip = ip
        self.bind_port = bind_port
        self.proc_count = proc_count
        self.log_path = log_path
        self.status = status
        self.started_at = datetime.datetime.now()

    def __repr__(self):
        return '<Worker %r>' % self.ip

class Resource(db.Model):
    id = db.Column(db.Integer, autoincrement='auto', primary_key=True)
    host_id = db.Column(db.String(36), db.ForeignKey('worker.id'))
    cpu_count = db.Column(db.Integer)
    cpu_usage = db.Column(db.Float)
    mem_usage = db.Column(db.Float)
    disk_usage = db.Column(db.Float)
    updated_at = db.Column(db.DateTime, onupdate=datetime.datetime.now)

    def __init__(self, worker=None, cpu_count=None, cpu_usage=None, mem_usage=None, disk_usage=None):
        self.host = worker
        self.cpu_count = cpu_count
        self.cpu_usage = cpu_usage
        self.mem_usage = mem_usage
        self.disk_usage = disk_usage
        self.updated_at = datetime.datetime.now()

    def __repr__(self):
        return '<Resource %r>' % self.ip

class Crash(db.Model):
    id = db.Column(db.Integer, autoincrement='auto', primary_key=True)
    host_id = db.Column(db.String(36), db.ForeignKey('worker.id'))
    crash_path = db.Column(db.String(4096))
    crash_name = db.Column(db.String(255))
    case_path = db.Column(db.String(4096))
    case_name = db.Column(db.String(255))
    status = db.Column(db.Integer)
    created_at = db.Column(db.DateTime)

    def __init__(self, worker=None, crash_path=None, case_path=None, status=CRASH_UNCHECKED):
        self.host = worker
        self.crash_path = crash_path
        self.crash_name = os.path.basename(self.crash_path)
        self.case_path = case_path
        self.case_name = os.path.basename(self.case_path)
        self.status = status
        self.created_at = datetime.datetime.now()

    def __repr__(self):
        return '<Crash %r>' % self.ip

class IokitOpen(db.Model):
    id = db.Column(db.Integer, autoincrement='auto', primary_key=True)
    service_name = db.Column(db.String(255))
    type = db.Column(db.Integer)
    class_name = db.Column(db.String(255))
    xnu_version_start = db.Column(db.Integer)
    submit_time = db.Column(db.DateTime)
    verify_open = db.Column(db.String(255))

    def __init__(self, service_name=None, type=None, class_name=None, xnu_version_start=None):
        self.service_name = service_name
        self.type = type
        self.class_name = class_name
        self.xnu_version_start = xnu_version_start
        self.submit_time = datetime.datetime.now()

    def __repr__(self):
        return '<IokitOpen %r>' % self.id

class IokitConnect(db.Model):
    id = db.Column(db.Integer, autoincrement='auto', primary_key=True)
    class_name = db.Column(db.String(255))
    selector = db.Column(db.Integer)

    scalar_input_count = db.Column(db.Integer)
    scalar_output_count = db.Column(db.Integer)
    struct_input_size = db.Column(db.Integer)
    struct_output_size = db.Column(db.Integer)

    ool_input_size = db.Column(db.Integer)
    ool_output_size = db.Column(db.Integer)
    md5_no_content = db.Column(db.String(255))
    xnu_version_start = db.Column(db.Integer)
    xnu_version_end = db.Column(db.Integer)

    submit_time = db.Column(db.DateTime)
    most_success = db.Column(db.String(255))
    fuzzing_round = db.Column(db.Integer)
    fuzzing_success_rate = db.Column(db.Float)

    def __init__(self, class_name=None, selector=None, scalar_input_count=None, scalar_output_count=None,
                 struct_input_size=None, struct_output_size=None,
                 ool_input_size=None, ool_output_size=None, md5_no_content=None, xnu_version_start=None,
                 xun_version_end=None, most_success=None, fuzzing_round=None, fuzzing_success_rate=None):
        self.class_name = class_name
        self.selector = selector
        self.scalar_input_count = scalar_input_count
        self.scalar_output_count = scalar_output_count
        self.struct_input_size = struct_input_size
        self.struct_output_size = struct_output_size
        self.ool_input_size = ool_input_size
        self.ool_output_size = ool_output_size
        self.md5_no_content = md5_no_content
        self.xnu_version_start = xnu_version_start
        self.xun_version_end = xun_version_end
        self.most_success = most_success
        self.fuzzing_round = fuzzing_round
        self.fuzzing_success_rate = fuzzing_success_rate
        self.submit_time = datetime.datetime.now()

    def __repr__(self):
        return '<IokitConnect %r>' % self.id

######################
class Record(db.Model):
    id = db.Column(db.Integer, autoincrement='auto', primary_key=True)
    base = db.Column(db.Float)
    beishu = db.Column(db.Float)
    code = db.Column(db.String(255))
    buy_date = db.Column(db.DateTime)
    follow = db.Column(db.Float)
    jingzhi = db.Column(db.Float)
    gushu = db.Column(db.Float)

    submit_time = db.Column(db.DateTime)

    def __init__(self, id, date, base, beishu, follow, code):
        self.id = id
        self.base = float(base)
        self.beishu = float(beishu)
        self.code = code
        self.buy_date = date if date else (datetime.date.today()).strftime('%Y-%m-%d')
        self.follow = float(follow) if follow else self.base * float(beishu)
        self.jingzhi = float(get_jingzhi(code, self.buy_date))
        self.gushu = round(float(self.follow) / float(self.jingzhi) if self.jingzhi != -1 else -1, 4)
        self.submit_time = datetime.datetime.now()

    def __repr__(self):
        return '<record %r>' % self.id


import tensorflow as tf
import tensorflow_quantum as tfq

import cirq
import sympy
import numpy as np
import collections

from utils.quantum_utils import convert_to_angle
from utils.te_layer_tf import EmbeddingTE

class CircuitLayerBuilder():
    def __init__(self, data_qubits, readout):
        self.data_qubits = data_qubits
        self.readout = readout

    def add_layer(self, circuit, gate, prefix):
        for i, qubit in enumerate(self.data_qubits):
            symbol = sympy.Symbol(prefix + '-' + str(i))
            circuit.append(gate(qubit, self.readout)**symbol)

def filter_36(x, y):
    keep = (y == 3) | (y == 6)
    x, y = x[keep], y[keep]
    y = y == 3
    return x,y

def remove_contradicting(xs, ys):
    mapping = collections.defaultdict(set)
    # Determine the set of labels for each unique image:
    for x,y in zip(xs,ys):
       mapping[tuple(x.flatten())].add(y)

    new_x = []
    new_y = []
    for x,y in zip(xs, ys):
      labels = mapping[tuple(x.flatten())]
      if len(labels) == 1:
          new_x.append(x)
          new_y.append(list(labels)[0])
      else:
          # Throw out images that match more than one label.
          pass

    num_3 = sum(1 for value in mapping.values() if True in value)
    num_6 = sum(1 for value in mapping.values() if False in value)
    num_both = sum(1 for value in mapping.values() if len(value) == 2)

    print("Number of unique images:", len(mapping.values()))
    print("Number of 3s: ", num_3)
    print("Number of 6s: ", num_6)
    print("Number of contradictory images: ", num_both)
    print()
    print("Initial number of examples: ", len(xs))
    print("Remaining non-contradictory examples: ", len(new_x))

    return np.array(new_x), np.array(new_y)

def qrac(data):
    data = data.reshape(-1).tolist()
    data += [0] * ((3-len(data)) % 3)

    var_list = []
    for i in range(0, len(data), 3):
        var_list += convert_to_angle(data[i:i+3])

    return var_list

def TE_31(data):
    data = data.reshape(-1).tolist()
    data += [0] * ((3-len(data)) % 3)

    var_list = []
    for i in range(0, len(data), 3):
        encode_dat = data[i:i+3]
        bt = ''
        for x in encode_dat:
            bt += str(x)

        var_list += [int(bt, 2)]

    return var_list

def conv(data):
    data = np.squeeze(data)

    var_list = []
    for i in range(4):
        for j in range(2):
            encode_dat = data[i, j:j+3].reshape(-1)
            var_list += convert_to_angle(encode_dat)

    dataT = data.T
    for i in range(4):
        for j in range(2):
            encode_dat = dataT[i, j:j+3].reshape(-1)
            var_list += convert_to_angle(encode_dat)

    return var_list

def conv_TE(data):
    data = np.squeeze(data)

    var_list = []
    for i in range(4):
        for j in range(2):
            encode_dat = data[i, j:j+3].reshape(-1)
            bt = ''
            for x in encode_dat:
                bt += str(x)

            var_list += [int(bt, 2)]

    dataT = data.T
    for i in range(4):
        for j in range(2):
            encode_dat = dataT[i, j:j+3].reshape(-1)
            bt = ''
            for x in encode_dat:
                bt += str(x)

            var_list += [int(bt, 2)]

    return var_list

def conv_41(data):
    var_list = []
    for i in range(3):
        for j in range(3):
            encode_dat = data[i:i+2, j:j+2].reshape(-1)
            bt = ''
            for x in encode_dat:
                bt += str(x)

            var_list += [int(bt, 2)]

    return var_list

def create_quantum_model(METHOD, num_qubit, LAYER):
    """Create a QNN model circuit and readout operation to go along with it."""
    if METHOD == '16px':
        data_qubits = cirq.GridQubit.rect(4,4)  # a 4x4 grid.
    elif METHOD == '8px':
        data_qubits = cirq.GridQubit.rect(2,4)  # a 2x4 grid.
    else:
        data_qubits = cirq.GridQubit.rect(num_qubit,1)  # a num_qubitx1 grid.
    readout = cirq.GridQubit(-1, -1)         # a single qubit at [-1,-1]
    circuit = cirq.Circuit()

    # Prepare the readout qubit.
    circuit.append(cirq.X(readout))
    circuit.append(cirq.H(readout))

    builder = CircuitLayerBuilder(
        data_qubits = data_qubits,
        readout=readout)

    # Then add layers (experiment by adding more).
    for i in range(LAYER):
        builder.add_layer(circuit, cirq.XX, f"xx{i+1}")
        builder.add_layer(circuit, cirq.ZZ, f"zz{i+1}")

    # Finally, prepare the readout qubit.
    circuit.append(cirq.H(readout))

    return circuit, cirq.Z(readout)

def create_normal_model(METHOD, num_qubit, LAYER):
    model_circuit, model_readout = create_quantum_model(METHOD, num_qubit, LAYER)
    model = tf.keras.Sequential([
        # The input is the data-circuit, encoded as a tf.string
        tf.keras.layers.Input(shape=(), dtype=tf.string),
        # The PQC layer returns the expected value of the readout gate, range [-1,1].
        tfq.layers.PQC(model_circuit, model_readout),
        tf.keras.layers.Lambda(lambda x: (x+1)/2),
    ])

    return model

def create_TE_model(METHOD, num_qubit, LAYER):
    model_circuit, model_readout = create_quantum_model(METHOD, num_qubit, LAYER)
    inp = tf.keras.layers.Input((num_qubit,))
    outputs = []
    for i in range(num_qubit):
        if METHOD in ['TE41', 'conv_41', 'TE41_dup', 'conv_41_s2']:
            outputs.append(tf.keras.layers.Embedding(16, 2)(inp[:,i]))
        else:
            outputs.append(tf.keras.layers.Embedding(8, 2)(inp[:,i]))

    outputs = tf.keras.layers.Concatenate()(outputs)
    embed_angle = tf.keras.layers.Concatenate()([outputs[:,::2], outputs[:,1::2]])

    # embed_angle = tf.keras.layers.Lambda(tf.squeeze)(embed_angle)
    output = EmbeddingTE(model_circuit, num_qubit, model_readout, initializer=tf.keras.initializers.RandomNormal(0, 2 * np.pi))(embed_angle)

    output = tf.keras.layers.Lambda(lambda x: (x + 1) / 2)(output)

    model = tf.keras.Model(inp, output)

    return model

def convert_to_circuit(image):
    """Encode truncated classical image into quantum datapoint."""
    values = np.ndarray.flatten(image)
    qubits = cirq.GridQubit.rect(4, 4)
    circuit = cirq.Circuit()
    for i, value in enumerate(values):
        if value:
            circuit.append(cirq.X(qubits[i]))
    return circuit

def convert_to_circuit_8px(image):
    """Encode truncated classical image into quantum datapoint."""
    values = np.ndarray.flatten(image[1:3,:])
    qubits = cirq.GridQubit.rect(2, 4)
    circuit = cirq.Circuit()
    for i, value in enumerate(values):
        if value:
            circuit.append(cirq.X(qubits[i]))
    return circuit

def convert_to_circuit_QRAC(data, num_qubit):
    """Encode truncated classical image by QRAC(3,1)."""
    values = data
    qubits = cirq.GridQubit.rect(num_qubit,1)
    circuit = cirq.Circuit()
    for i in range(len(data) // 2):
        circuit.append(cirq.rx(data[2*i])(qubits[i]))
        circuit.append(cirq.ry(data[2*i+1])(qubits[i]))
    return circuit

def scheduler(epoch, lr):
    if epoch > 5:
        return 1e-3
    else:
        return 1e-2


from tensorflow import keras
from sklearn.preprocessing import MinMaxScaler
import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import os

def save_fig(fig_id, Images_path, tight_layout=False, fig_extension="pdf", resolution=300):
    path = os.path.join(Images_path, fig_id + "." + fig_extension)
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format=fig_extension, dpi=resolution)

def plot_series(series, n_steps, y=None, y_pred=None, x_label="Trading Days", y_label="Closing Price", legend=True):
    plt.plot(series,".-",color='steelblue')
    if y is not None:
        plt.plot(n_steps, y, ".-",color='lightskyblue', label="Target")
    if y_pred is not None:
        plt.plot(n_steps, y_pred, "x-",color='orange', markersize=5, label="Prediction")
    plt.grid(True)
    if x_label:
        plt.xlabel(x_label, fontsize=10)
    if y_label:
        plt.ylabel(y_label, fontsize=10)
    plt.hlines(0, 0, 100, linewidth=1)
    plt.axis([0, n_steps + 1, 0, 1])
    if legend and (y or y_pred):
        plt.legend(fontsize=10, loc="upper left")

def plot_learning_curves(loss, val_loss):
    plt.plot(np.arange(len(loss)) + 0.5, loss, "b.-", label="Training loss")
    plt.plot(np.arange(len(val_loss)) + 1, val_loss, "r.-", label="Validation loss")
    plt.gca().xaxis.set_major_locator(mpl.ticker.MaxNLocator(integer=True))
    plt.axis([1, 20, 0, 0.05])
    plt.legend(fontsize=14)
    plt.xlabel("Epochs")
    plt.ylabel("Loss")
    plt.grid(True)

def plot_multiple_forecasts_3D(X, Y, Y_pred):
    n_steps = X.shape[1]
    ahead = Y.shape[1]
    plot_series(X[0, :, 0], n_steps)
    plt.plot(np.arange(n_steps, n_steps + ahead), Y[0, :, 0], ".-",color='lightskyblue', label="Actual")
    plt.plot(np.arange(n_steps, n_steps + ahead), Y_pred[0, :, 0], "x-",color='orange', label="Forecast", markersize=5)
    plt.axis([n_steps - 5, n_steps + ahead, 0, 200])
    plt.legend(fontsize=10)

def plot_multiple_forecasts_3D_test(X, Y, Y_pred):
    n_steps = X.shape[1]
    ahead = Y.shape[2]
    iterations = [0,60,120,180]
    plot_series(X[0, :, 0], n_steps)
    for i in iterations:
        plt.plot(np.arange(n_steps + i, n_steps + ahead + i), Y[i, -1, :], ".-",color='lightskyblue', label="Actual")
        plt.plot(np.arange(n_steps + i, n_steps + ahead + i), Y_pred[i, :, :], "x-",color='orange', label="Forecast", markersize=5)
    plt.axis([n_steps - 5, 500, 0, 2])
    plt.legend(fontsize=10)

def plot_multiple_forecasts_2D(X, Y, Y_pred):
    n_steps = X.shape[1]
    ahead = Y.shape[1]
    plot_series(X[0, :], n_steps)
    plt.plot(np.arange(n_steps, n_steps + ahead), Y[0, :], ".-",color='lightskyblue', label="Actual")
    plt.plot(np.arange(n_steps, n_steps + ahead), Y_pred[0, :], "x-",color='orange', label="Forecast", markersize=5)
    plt.axis([n_steps - 5, n_steps + ahead, 0, 200])
    plt.legend(fontsize=10)

def last_time_step_mse(Y_true, Y_pred):
    return keras.metrics.mean_squared_error(Y_true[:, -1], Y_pred[:, -1])

def create_sets_vector(db_scaled, n_steps, t_steps):
    new_series = []
    for row in range(len(db_scaled)-n_steps-t_steps+1):
        ser = db_scaled[row:row+n_steps+t_steps]
        new_series.append(ser)

    new_series = np.array(new_series)
    new_series = new_series.reshape(new_series.shape[0], new_series.shape[1], 1)

    len_set = len(new_series)
    len_train = int(2/3 * len_set)
    len_val = int(5/6 * len_set)
    len_test = len_set - 1

    X_train, Y_train = new_series[:len_train, :n_steps], new_series[:len_train,-t_steps:-1,0]
    X_val, Y_val = new_series[len_train:len_val, :n_steps], new_series[len_train:len_val, -t_steps:,0]
    X_test, Y_test = new_series[len_val:len_test, :n_steps], new_series[len_val:len_test, -t_steps:,0]
    X_new, Y_new = new_series[-1:,:n_steps], new_series[len_test:,-t_steps:,0]

    return X_train, Y_train, X_val, Y_val, X_test, Y_test, X_new, Y_new

def split_set(db):
    len_set = len(db)
    len_train = int(2/3 * len_set)
    len_val = int(5/6 * len_set)

    train, val, test = db[:len_train], db[len_train:len_val], db[len_val:]
    train = np.array(train).reshape(-1,1)
    val = np.array(val).reshape(-1,1)
    test = np.array(test).reshape(-1,1)
    return train, val, test

def preprocess_data(db_scaled,n_steps,t_steps):
    new_series = []
    for row in range(len(db_scaled)-n_steps-t_steps+1):
        ser = db_scaled[row:row+n_steps+t_steps]
        new_series.append(ser)

    new_series = np.array(new_series)
    new_series = new_series.reshape(new_series.shape[0], new_series.shape[1], 1)

    len_set = len(new_series)

    Y = np.empty((len_set, n_steps, t_steps))
    for step_ahead in range(1, t_steps + 1):
        Y[ : , : ,step_ahead - 1] = new_series[ : , step_ahead:step_ahead + n_steps, 0]

    X = new_series[:,:n_steps]

    return  X, Y

def create_sets_sequence_sc(db, n_steps, t_steps):

    len_set = len(db)
    len_train = int(2/3 * len_set)
    len_val = int(5/6 * len_set)
    len_test = len_set

    print(len_set)
    print(len_train)
    print(len_val)
    print(len_test)

    train = db[:len_train]
    val = db[len_train:len_val]
    test = db[len_val:len_test]

    scaler = MinMaxScaler()
    train = scaler.fit_transform(train)
    val = scaler.transform(val)
    test = scaler.transform(test)

    #train, mean_train, std_train = transf_data(train)
    #val, mean_val, std_val = transf_data(val)
    #test, mean_test, std_test = transf_data(test)

    def preprocess_data(db_scaled):
        new_series = []
        for row in range(len(db_scaled)-n_steps-t_steps+1):
            ser = db_scaled[row:row+n_steps+t_steps]
            new_series.append(ser)

        new_series = np.array(new_series)
        new_series = new_series.reshape(new_series.shape[0], new_series.shape[1], 1)

        len_set = len(new_series)

        Y = np.empty((len_set, n_steps, t_steps))
        for step_ahead in range(1, t_steps + 1):
            Y[ : , : ,step_ahead - 1] = new_series[ : , step_ahead:step_ahead + n_steps, 0]

        X = new_series[:,:n_steps]

        return  X, Y

    X_train, Y_train = preprocess_data(train)
    X_val, Y_val = preprocess_data(val)
    X_test, Y_test = preprocess_data(test)

    return X_train, Y_train, X_val, Y_val, X_test, Y_test, #mean_test, #std_test

def create_sets_sequence(db_scaled, n_steps, t_steps):
    new_series = []
    for row in range(len(db_scaled)-n_steps-t_steps+1):
        ser = db_scaled[row:row+n_steps+t_steps]
        new_series.append(ser)

    new_series = np.array(new_series)
    new_series = new_series.reshape(new_series.shape[0], new_series.shape[1], 1)

    len_set = len(new_series)
    len_train = int(2/3 * len_set)
    len_val = int(5/6 * len_set)
    len_test = len_set - 1

    Y = np.empty((len_set, n_steps, t_steps))
    for step_ahead in range(1, t_steps + 1):
        Y[ : , : ,step_ahead - 1] = new_series[ : , step_ahead:step_ahead + n_steps, 0]

    X_train, Y_train = new_series[:len_train, :n_steps], Y[:len_train]
    X_val, Y_val = new_series[len_train:len_val, :n_steps], Y[len_train:len_val]
    X_test, Y_test = new_series[len_val:len_test, :n_steps], Y[len_val:len_test]
    X_new, Y_new = new_series[-1:,:n_steps], new_series[len_test:,n_steps:]

    return X_train, Y_train, X_val, Y_val, X_test, Y_test, X_new, Y_new

def read_data(file_input):
    dataset = pd.read_csv(file_input, skiprows=1)
    dataset.columns = ['Date', 'Close']

    dataset = dataset.dropna(how='any')

    db = dataset['Close']
    return db

def transf_data(db):
    #db = np.log(db) / np.array(10)
    #return db
    db = np.log(db)
    mean_db = np.mean(db)
    std_db = np.std(db)
    new_db = (db - np.array(mean_db)) / np.array(std_db)

    return new_db, mean_db, std_db

def inv_tranf_data(output, mean_db, std_db):
    #output = np.array(10) * output
    #output = np.exp(output)
    #return output

    output = output * np.array(std_db)
    output = output + mean_db
    output = np.exp(output)

    return output

def inv_tranf_data_sc(output, mean_db, std_db):

    pass

class MCDropout(keras.layers.Dropout):
    def call(self, inputs):
        return super().call(inputs, training=True)

    def get_config(self):
        return {
        'rate': self.rate,
        'noise_shape': self.noise_shape,
        'seed': self.seed
         }

    @classmethod
    def from_config(cls, config):
        return cls(**config)

class GatedActivationUnit(keras.layers.Layer):
    def __init__(self, activation="tanh", **kwargs):
        super().__init__(**kwargs)
        self.activation = keras.activations.get(activation)
    def call(self, inputs):
        n_filters = inputs.shape[-1] // 2
        linear_output = self.activation(inputs[..., :n_filters])
        gate = keras.activations.sigmoid(inputs[..., n_filters:])
        return self.activation(linear_output) * gate

def wavenet_residual_block(inputs, n_filters, dilation_rate):
    z = keras.layers.Conv1D(2 * n_filters, kernel_size=2, padding="causal",
                            dilation_rate=dilation_rate)(inputs)
    z = GatedActivationUnit()(z)
    z = keras.layers.Conv1D(n_filters, kernel_size=1)(z)
    return keras.layers.Add()([z, inputs]), z


"""This extension sets the bot's playing status."""

import asyncio
import discord

class PlayingStatus:
    """Bot playing status manager."""

    def __init__(self, bot):
        self.bot = bot
        self.statuses = []
        self.index = 0

        #this must be created as task bc it must be checked after specific intervals
        self.playing_status_task = self.bot.loop.create_task(self.update_playing_status())

    def get_next_status(self):
        """Get next playing status type."""

        status = self.statuses[self.index]

        if isinstance(status, tuple):
            #mind fucking and super fancy way of calling functions
            status = status[0].format(status[1]())

        self.index += 1

        if self.index >= len(self.statuses):
            self.index = 0

        return status

    def guild_count(self):
        """Get guild count from bot."""

        guilds = len(self.bot.guilds)
        return guilds

    def member_count(self):
        """Get human users count from bot."""

        member_count = sum(not m.bot for m in self.bot.get_all_members())
        return member_count

    async def update_playing_status(self):
        """Every 10 seconds, update the bot's playing status."""

        await self.bot.wait_until_ready()

        display_prefix = self.bot.config['prefix']
        if len(display_prefix) > 1:
            display_prefix += " " #extra space appended for bigger prefixes

        self.statuses = [
            f"'{display_prefix}help' for help.",
            f"'{display_prefix}info' for info.",
            ("in {0} servers", self.guild_count),
            ("with {0} Pewdiepie fans", self.member_count),
        ]

        while True:
            status = self.get_next_status()
            activity=discord.Streaming(name=status, url="https://www.twitch.tv/pewdiepie")
            await self.bot.change_presence(activity=activity)
            await asyncio.sleep(10)

def setup(bot):
    """Set up the extension."""
    bot.add_cog(PlayingStatus(bot))


import inspect
from functools import wraps
from typing import Callable, List, Optional, Type, Union

import attr
from django.http import HttpRequest, HttpResponse
from django.urls import include, path as url_path
from django.urls.resolvers import URLPattern
from django.utils.functional import cached_property
from django.views import View
from django.views.decorators.http import require_http_methods

from apirouter.conf import (
    get_default_exception_handler,
    get_default_request_class,
    get_default_response_class,
)
from apirouter.decorators import compose_decorators
from apirouter.request import Request
from apirouter.types import ExceptionHandlerType, RequestType
from apirouter.utils import removeprefix

@attr.dataclass(frozen=True)
class APIViewFuncRoute:
    path: str
    view_func: Callable
    view_kwargs: Optional[dict] = None
    methods: Optional[List[str]] = None
    name: Optional[str] = None
    request_class: Optional[Type[RequestType]] = None

    def __attrs_post_init__(self):
        object.__setattr__(self, "path", removeprefix(self.path, prefix="/"))
        if self.methods:
            view_func = require_http_methods(self.methods)(self.view_func)
            object.__setattr__(self, "view_func", view_func)

@attr.dataclass(frozen=True)
class APIViewClassRoute:
    path: str
    view: Union[Type[View], Callable]
    view_func: Callable = attr.ib(init=False)
    view_kwargs: Optional[dict] = None
    name: Optional[str] = None
    decorators: Optional[List[Callable]] = None
    request_class: Optional[Type[RequestType]] = None

    def __attrs_post_init__(self):
        object.__setattr__(self, "path", removeprefix(self.path, prefix="/"))
        if inspect.isclass(self.view):
            view_func = self.view.as_view()
        else:
            view_func = self.view
        if self.decorators:
            view_func = compose_decorators(*self.decorators)(view_func)
        object.__setattr__(self, "view_func", view_func)

@attr.dataclass(frozen=True)
class APIIncludeRoute:
    router: "APIRouter"
    prefix: str = ""

APIRoute = Union[APIViewFuncRoute, APIViewClassRoute]
APIRouteAny = Union[APIViewFuncRoute, APIViewClassRoute, APIIncludeRoute]

class APIRouter:
    def __init__(
        self,
        *,
        name: Optional[str] = None,
        decorators: Optional[List[Callable]] = None,
        exception_handler: Optional[ExceptionHandlerType] = None,
        request_class: Optional[Type[RequestType]] = None,
        response_class: Optional[Type[HttpResponse]] = None,
    ):
        self.name = name
        self.decorators = decorators or []
        self.exception_handler = exception_handler or get_default_exception_handler()
        self.request_class = request_class or get_default_request_class()
        self.response_class = response_class or get_default_response_class()
        self.routes: List[APIRouteAny] = []

    @cached_property
    def urls(self) -> List[URLPattern]:
        urls = self._build_urls()
        if self.name:
            return [url_path("", include((urls, self.name)))]
        return urls

    def include_router(self, router: "APIRouter", *, prefix: str = "") -> None:
        if prefix:
            prefix = removeprefix(prefix, prefix="/")
        self.routes.append(APIIncludeRoute(router=router, prefix=prefix))

    def add_route(
        self,
        path: str,
        view_func: Callable,
        *,
        view_kwargs: Optional[dict] = None,
        name: Optional[str] = None,
        methods: Optional[List[str]] = None,
        request_class: Optional[Type[RequestType]] = None,
    ) -> None:
        self.routes.append(
            APIViewFuncRoute(
                path=path,
                view_func=view_func,
                view_kwargs=view_kwargs,
                name=name,
                methods=methods,
                request_class=request_class,
            )
        )

    def add_view(
        self,
        path: str,
        view_class: Type[View],
        *,
        view_kwargs: Optional[dict] = None,
        name: Optional[str] = None,
        decorators: Optional[List[Callable]] = None,
        request_class: Optional[Type[RequestType]] = None,
    ) -> None:
        self.routes.append(
            APIViewClassRoute(
                path=path,
                view=view_class,
                view_kwargs=view_kwargs,
                name=name,
                decorators=decorators,
                request_class=request_class,
            )
        )

    def route(
        self,
        path: str,
        *,
        view_kwargs: Optional[dict] = None,
        methods: Optional[List[str]] = None,
        name: Optional[str] = None,
        request_class: Optional[Type[RequestType]] = None,
    ) -> Callable:
        def decorator(view_func: Callable):
            self.add_route(
                path,
                view_func,
                view_kwargs=view_kwargs,
                name=name,
                methods=methods,
                request_class=request_class,
            )
            return view_func

        return decorator

    def view(
        self,
        path: str,
        *,
        view_kwargs: Optional[dict] = None,
        name: Optional[str] = None,
        decorators: Optional[List[Callable]] = None,
        request_class: Optional[Type[RequestType]] = None,
    ) -> Callable:
        def decorator(view_class: Type[View]) -> Callable:
            self.add_view(
                path,
                view_class,
                view_kwargs=view_kwargs,
                name=name,
                decorators=decorators,
                request_class=request_class,
            )
            return view_class

        return decorator

    def path(
        self,
        route: str,
        view: Callable,
        *,
        kwargs: Optional[dict] = None,
        name: Optional[str] = None,
    ) -> URLPattern:
        """
        Django compatible path method.
        """
        request_class: Optional[Type[RequestType]]

        if hasattr(view, "view_class"):
            view_class: Type[View] = getattr(view, "view_class")
            request_class = getattr(view_class, "__apirouter_request_class", None)
            decorators: Optional[List[Callable]] = getattr(
                view_class, "__apirouter_decorators", None
            )
            return self._path_route(
                APIViewClassRoute(
                    path=route,
                    view=view,
                    view_kwargs=kwargs,
                    name=name,
                    request_class=request_class,
                    decorators=decorators,
                )
            )

        request_class = getattr(view, "__apirouter_request_class", None)

        return self._path_route(
            APIViewFuncRoute(
                path=route,
                view_func=view,
                view_kwargs=kwargs,
                name=name,
                request_class=request_class,
            )
        )

    def _path_route(self, route: APIRoute) -> URLPattern:
        """
        Make route URL pattern.
        """
        return url_path(
            route.path,
            view=self._handle(route),
            kwargs=route.view_kwargs,
            name=route.name,
        )

    def _include_route(self, route: APIIncludeRoute) -> URLPattern:
        """
        Make include URL pattern.
        """
        return url_path(route.prefix, include(route.router.urls))

    def _build_urls(self) -> List[URLPattern]:
        """
        Build Django URL patterns sequence.
        """
        urlpatterns: List[URLPattern] = []

        for route in self.routes:
            if isinstance(route, APIIncludeRoute):
                urlpatterns.append(self._include_route(route))
            else:
                urlpatterns.append(self._path_route(route))

        return urlpatterns

    def _handle(self, route: APIRoute) -> Callable:
        """
        Handle route.
        """
        request_class = route.request_class or self.request_class
        return self._handle_view(route.view_func, request_class=request_class)

    def _handle_view(
        self, view: Callable, request_class: Type[RequestType]
    ) -> Callable:
        """
        Handle view.
        """

        @wraps(view)
        def wrapped_view(request: HttpRequest, *args, **kwargs) -> HttpResponse:
            if issubclass(request_class, Request):
                request = request_class(request)
            try:
                get_response = compose_decorators(*self.decorators)(view)
                response = get_response(request, *args, **kwargs)
                if not isinstance(response, HttpResponse):
                    return self.response_class(response)
                return response
            except Exception as exc:
                return self.exception_handler(request, exc)

        return wrapped_view


# Exercise 8 - Datetime.py
# Mark Cotter, V1_05, 2019-03-30

# V1_05 - 2019-03-30
# Comments updated

# V1_04 - 2019-03-23
# Variables initial values added
# Minor comments added

# V1_03 - 2019-03-22
# Amendment made to day of month suffix.
# Comments added

# V1_02 - 2019-03-22
# Added functions to change day of month suffix to display, display PM/AM as
# lower case and remove unnecessary 0 from the text.

# V1_01 - Program created 2019-03-21

# A program that that outputs today's date and time in the format
# "Monday, January 10th 2019 at 1:15pm".

# Import the datetime module abbreviated to 'dt'
# Note the name of this program has to have a capital letter 'Datetime.py'
# as the function 'datetime' will not work if the program name is the same i.e.
# 'datetime.py'
# Code adapted from Week 6 lecture
import datetime as dt

# Initial variable values for 'year' 'month' 'day' 'suffix' 'weekday' 'hour'
# 'minute' and 'ampm'
year = 2019
month = "January"
day = 1
suffix = "st"
weekday = "Monday"
hour = 0
minute = "00"
ampm = "am"

# Set variable 't' to current time
# Code adapted for printing the various year month day and time from webpage
# https://docs.python.org/3/library/datetime.html#datetime.datetime.fromtimestamp
# and Exercise 2 - begins-with-t.py

# Set 'year' to current year, int because will always be 4 digits
year = int(dt.datetime.now().strftime("%Y"))
# Set 'month' to current month, str because is a word
month = str(dt.datetime.now().strftime("%B"))
# Set 'day' to current day of the month, int for reviewing suffix & removing lead 0
day = int(dt.datetime.now().strftime("%d"))
# Set 'weekday' to current day of the week, str because is a word
weekday = str(dt.datetime.now().strftime("%A"))
# Set 'hour' to current hour, int to remove leading zero
hour = int(dt.datetime.now().strftime("%I"))
# Set 'minute' to current minute, str because always want 2 digits
minute = str(dt.datetime.now().strftime("%M"))
# Set 'ampm' to current 12 clock morning or evening i.e. 'AM' or 'PM'
ampm = str(dt.datetime.now().strftime("%p"))

# Set AM or PM to lower case
# Code adapted from Week 7 lecture
ampm = ampm.lower()

# Temporary variable setting for testing suffix changes
#day = 31

# Add suffix to the day of the month
if day == 1:
    # Add abbreviated suffix for first
    suffix = "st"
elif day == 2:
    # Add abbreviated suffix for second
    suffix = "nd"
elif day == 3:
    # Add abbreviated suffix for third
    suffix = "rd"
elif day == 21:
    # Add abbreviated suffix for twenty first
    suffix = "st"
elif day == 22:
    # Add abbreviated suffix for twenty second
    suffix = "nd"
elif day == 23:
    # Add abbreviated suffix for twenty third
    suffix = "rd"
elif day == 31:
    # Add abbreviated suffix for thirty first
    suffix = "st"
else:
    suffix = "th"

# Prints the result
print(f"{weekday}, {month} {day}{suffix} {year} at {hour}:{minute}{ampm}")

# Code reference sources:
# 1. Dr Ian McLoughlin, GMIT: H Dip in Data Analytics lecture notes,
#    I adapted code from the Week 6 lecture for importing datetime module
#
# 2. Code adapted for printing the various year month day and time from webpage
#    https://docs.python.org/3/library/datetime.html#datetime.datetime.fromtimestamp
#    and Exercise 2 - begins-with-t.py
#
# 3. Code adapted from Week 7 lecture for changing string text to lowercase


import ConfigParser
import json
class Config(object):
    """Holds model hyperparams and data information.

    The config class is used to store various hyperparameters and dataset
    information parameters. Model objects are passed a Config() object at
    instantiation.
    """
    """General"""
    train_data = './all_data/train.txt'
    val_data = './all_data/valid.txt'
    test_data = './all_data/test.txt'
    vocab_path = './all_data/vocab.100k'
    embed_path = './all_data/embed/embedding.'

    sent_rep = "lstm"
    pre_trained = True
    batch_size = 128
    embed_size = 100
    max_epochs = 100
    early_stopping = 5
    dropout = 0.9
    lr = 0.3
    decay_epoch = 1.0
    decay_rate = 0.95
    reg = 1e-5
    num_steps = 100
    beam_size = 1

    """lstm"""
    hidden_size = 100
    rnn_numLayers = 1

    """cnn"""
    num_filters = 512
    filter_sizes = [3, 4, 5]
    cnn_numLayers = 1

    def saveConfig(self, filePath):
        cfg = ConfigParser.ConfigParser()
        cfg.add_section('General')
        cfg.add_section('lstm')
        cfg.add_section('cnn')

        cfg.set('General', 'train_data', self.train_data)
        cfg.set('General', 'val_data', self.val_data)
        cfg.set('General', 'test_data', self.test_data)
        cfg.set('General', 'vocab_path', self.vocab_path)
        cfg.set('General', 'embed_path', self.embed_path)

        cfg.set('General', 'sent_rep', self.sent_rep)
        cfg.set('General', 'pre_trained', self.pre_trained)
        cfg.set('General', 'batch_size', self.batch_size)
        cfg.set('General', 'embed_size', self.embed_size)
        cfg.set('General', 'max_epochs', self.max_epochs)
        cfg.set('General', 'early_stopping', self.early_stopping)
        cfg.set('General', 'dropout', self.dropout)
        cfg.set('General', 'lr', self.lr)
        cfg.set('General', 'decay_epoch', self.decay_epoch)
        cfg.set('General', 'decay_rate',self.decay_rate)
        cfg.set('General', 'reg', self.reg)
        cfg.set('General', 'num_steps', self.num_steps)
        cfg.set('General', 'beam_size', self.beam_size)

        cfg.set('lstm', 'hidden_size', self.hidden_size)
        cfg.set('lstm', 'rnn_numLayers', self.rnn_numLayers)

        cfg.set('cnn', 'num_filters', self.num_filters)
        cfg.set('cnn', 'filter_sizes', self.filter_sizes)
        cfg.set('cnn', 'cnn_numLayers', self.cnn_numLayers)

        with open(filePath, 'w') as fd:
            cfg.write(fd)

    def loadConfig(self, filePath):
        cfg = ConfigParser.ConfigParser()
        cfg.read(filePath)

        self.train_data = cfg.get('General', 'train_data')
        self.val_data = cfg.get('General', 'val_data')
        self.test_data = cfg.get('General', 'test_data')
        self.vocab_path = cfg.get('General', 'vocab_path')
        self.embed_path = cfg.get('General', 'embed_path')

        self.sent_rep = cfg.get('General', 'sent_rep')
        self.pre_trained = cfg.getboolean('General', 'pre_trained')
        self.batch_size = cfg.getint('General', 'batch_size')
        self.embed_size = cfg.getint('General', 'embed_size')
        self.max_epochs = cfg.getint('General', 'max_epochs')
        self.early_stopping = cfg.getint('General', 'early_stopping')
        self.dropout = cfg.getfloat('General', 'dropout')
        self.lr = cfg.getfloat('General', 'lr')
        self.decay_epoch = cfg.getfloat('General', 'decay_epoch')
        self.decay_rate = cfg.getfloat('General', 'decay_rate')
        self.reg = cfg.getfloat('General', 'reg')
        self.num_steps = cfg.getint('General', 'num_steps')
        self.beam_size = cfg.getint('General', 'beam_size')

        self.hidden_size = cfg.getint('lstm', 'hidden_size')
        self.rnn_numLayers = cfg.getint('lstm', 'rnn_numLayers')

        self.num_filters = cfg.getint('cnn', 'num_filters')
        self.filter_sizes = json.loads(cfg.get('cnn', 'filter_sizes'))
        self.cnn_numLayers = cfg.getint('cnn', 'cnn_numLayers')


from django.shortcuts import render
import wikipedia
# Create your views here.

def wiki(request):
    if request.method=='POST':
        search=request.POST['search']
        print(search)
        search_results = wikipedia.search(search)
        if search_results:
            article_summary=wikipedia.summary(search)
            print(article_summary)
            return render(request,'wiki.html',{'data':article_summary})
        else:
           print("No search results found.")
    return render(request,'wiki.html')


from urllib import response
import requests

name = "ZOE TEO XUAN YU"
url = f'https://data.gov.sg/api/action/datastore_search?resource_id=a41ce851-728e-4d65-8dc5-e0515a01ff31&limit=5&q={name}'

response = requests.get(url)

if(response.status_code <= 200 & response.status_code >= 29):
    print(response.json())
    result = response.json()['result']['records']


import streamlit as st
from calculator import FuelCostCalculator

st.set_page_config(
    page_title="Fuel Cost Calculator",
    page_icon=":car:",
    layout="centered",
)

with st.container():
    # title
    st.write("# :car: Fuel Cost Calculator")

    # description
    st.write("Calculate fuel cost for every km you drive.")

    # input fields
    st.write("---")
    base_currency = st.selectbox("💵 Base currency", ("R", "$", "€", "£"))
    tank_size = float(st.text_input(label="💧 Fuel tank size [L]", value="40.0"))
    distance_on_tank = float(st.text_input(label="🌍 Distance you can drive on a full tank [km]", value="450.0"))
    fuel_cost = float(st.text_input(label=f"⛽ Fuel cost [{base_currency}/L]", value="24.38"))

    # calculate button
    st.write("---")
    if st.button("Calculate"):
        calc = FuelCostCalculator(
            tank_size=tank_size,
            distance_on_tank=distance_on_tank,
            fuel_cost=fuel_cost,
            base_currency=base_currency
        )
        cost = calc.calculate_cost_per_km()
        st.write(f"### Cost per km: {calc.base_currency} {round(cost, 2)}")
    else:
        st.write("*Click Calculate when ready.*")
    st.write("---")


import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
import warnings
from typing import Sequence

from numba import jit, prange

def leaf_samples(rf, X_not_col:np.ndarray) -> Sequence:
    """
    Return a list of arrays where each array is the set of X sample indexes
    residing in a single leaf of some tree in rf forest. For example, if there
    are 4 leaves (in one or multiple trees), we might return:

        array([array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
           array([10, 11, 12, 13, 14, 15]), array([16, 17, 18, 19, 20]),
           array([21, 22, 23, 24, 25, 26, 27, 28, 29]))
    """
    n_trees = len(rf.estimators_)
    leaf_samples = []
    leaf_ids = rf.apply(X_not_col)  # which leaf does each X_i go to for sole tree?
    for t in range(n_trees):
        # Group by id and return sample indexes
        uniq_ids = np.unique(leaf_ids[:,t])
        sample_idxs_in_leaves = [np.where(leaf_ids[:, t] == id)[0] for id in uniq_ids]
        leaf_samples.extend(sample_idxs_in_leaves)
    return leaf_samples

def partial_dependence(X:pd.DataFrame, y:pd.Series, colname:str,
                       min_slopes_per_x=5,
                       parallel_jit=True,
                       n_trees=1, min_samples_leaf=15, rf_bootstrap=False, max_features=1.0,
                       supervised=True,
                       verbose=False):
    """
    Internal computation of partial dependence information about X[colname]'s effect on y.
    Also computes partial derivative of y with respect to X[colname].

    :param X: Dataframe with all explanatory variables
    :param y: Series or vector with response variable
    :param colname: which X[colname] (a string) to compute partial dependence for
    :param min_slopes_per_x: ignore pdp y values derived from too few slopes; this is
           same count across all features (tried percentage of max slope count but was
           too variable). Important for getting good starting point of PD.

    Returns:
        leaf_xranges    The ranges of X[colname] partitions

        leaf_slopes     Associated slope for each leaf xrange

        slope_counts_at_x How many slopes are available at each x_j location

        dx              The change in x from one non-NaN unique X[colname] to the next

        slope_at_x      The slope at each non-NaN unique X[colname]

        pdpx            The non-NaN unique X[colname] values; len(pdpx)<=len(unique(X[colname]))

        pdpy            The effect of each non-NaN unique X[colname] on y; effectively
                        the cumulative sum (integration from X[colname] x to z for all
                        z in X[colname]). The first value is always 0.

        ignored         How many samples from len(X) total records did we have to
                        ignore because samples in leaves had identical X[colname]
                        values.
  """
    X_not_col = X.drop(colname, axis=1).values
    # For x floating-point numbers that are very close, I noticed that np.unique(x)
    # was treating floating-point numbers different in the 12th decimal point as different.
    # This caused a number of problems likely but I didn't notice it until I tried
    # np.gradient(), which found extremely huge derivatives. I fixed that with a hack:
    X_col = X[colname].values.round(decimals=10)

    if supervised:
        rf = RandomForestRegressor(n_estimators=n_trees,
                                   min_samples_leaf=min_samples_leaf,
                                   bootstrap=rf_bootstrap,
                                   max_features=max_features)
        rf.fit(X_not_col, y)
        if verbose:
            print(f"Strat Partition RF: dropping {colname} training R^2 {rf.score(X_not_col, y):.2f}")

    else:
        """
        Wow. Breiman's trick works in most cases. Falls apart on Boston housing MEDV target vs AGE
        """
        if verbose: print("USING UNSUPERVISED MODE")
        X_synth, y_synth = conjure_twoclass(X)
        rf = RandomForestClassifier(n_estimators=n_trees,
                                    min_samples_leaf=min_samples_leaf,
                                    bootstrap=rf_bootstrap,
                                    max_features=max_features,
                                    oob_score=False)
        rf.fit(X_synth.drop(colname, axis=1), y_synth)

    if verbose:
        leaves = leaf_samples(rf, X_not_col)
        nnodes = rf.estimators_[0].tree_.node_count
        print(f"Partitioning 'x not {colname}': {nnodes} nodes in (first) tree, "
              f"{len(rf.estimators_)} trees, {len(leaves)} total leaves")

    leaf_xranges, leaf_slopes, ignored = \
        collect_discrete_slopes(rf, X_col, X_not_col, y) # if ignored, won't have entries in leaf_* results

    # print('leaf_xranges', leaf_xranges)
    # print('leaf_slopes', leaf_slopes)

    real_uniq_x = np.unique(X_col)   # comes back sorted
    if verbose:
        print(f"discrete StratPD num samples ignored {ignored}/{len(X)} for {colname}")

    #print("uniq x =", len(real_uniq_x), "slopes.shape =", leaf_slopes.shape, "x ranges.shape", leaf_xranges.shape)
    if parallel_jit:
        slope_at_x, slope_counts_at_x = \
            avg_slopes_at_x_jit(real_uniq_x, leaf_xranges, leaf_slopes)
    else:
        slope_at_x, slope_counts_at_x = \
            avg_slopes_at_x_nonparallel_jit(real_uniq_x, leaf_xranges, leaf_slopes)

    if min_slopes_per_x <= 0:
        min_slopes_per_x = 1 # must have at least one slope value

    # Turn any slopes with weak evidence into NaNs but keep same slope_at_x length
    slope_at_x = np.where(slope_counts_at_x >= min_slopes_per_x, slope_at_x, np.nan)

    pdpx = real_uniq_x

    # At this point, slope_at_x will have at least one nan, but that's okay because
    # diff and nancumsum will treat them appropriately.  nancumsum treats NaN as 0
    # so the previous non-NaN value is carried forward until we reach next real value.
    # Keep in mind that the last slope in slope_at_x is always nan,
    # since we have no information beyond that. However, we can still produce
    # a pdpy value for that last position because we have a delta from the
    # previous position that will get us to the last position.

    # Integrate the partial derivative estimate in slope_at_x across pdpx to get dependence
    dx = np.diff(pdpx)      # for n pdpx values, there are n-1 dx values
    dydx = slope_at_x[:-1]  # for n y values, only n-1 slope values; last slope always nan

    y_deltas = dydx * dx                          # get change in y from x[i] to x[i+1]
    pdpy = np.nancumsum(y_deltas)                 # y_deltas has one less value than num x values
    pdpy = np.concatenate([np.array([0]), pdpy])  # align with x values; our PDP y always starts from zero

    # At this point pdpx, pdpy have the same length as real_uniq_x

    # Strip from pdpx,pdpy any positions for which we don't have useful slope info. If we have
    # slopes = [1, 3, nan] then cumsum will give 3 valid pdpy values. But, if we have
    # slopes = [1, 3, nan, nan], then there is no pdpy value for last position. nancumsum
    # carries same y to the right, but we wanna strip last position, in this case.
    # Detect 2 nans in a row. We don't want the second or any after that until next real value
    idx_not_adjacent_nans = np.where(~(np.isnan(slope_at_x[1:]) & np.isnan(slope_at_x[:-1])))[0] + 1
    # Deal with first position. If it's nan, drop it, else keep it
    if not np.isnan(slope_at_x[0]):
        idx_not_adjacent_nans = np.concatenate([np.array([0]), idx_not_adjacent_nans])
    pdpx = pdpx[idx_not_adjacent_nans]
    pdpy = pdpy[idx_not_adjacent_nans]

    return leaf_xranges, leaf_slopes, slope_counts_at_x, dx, slope_at_x, pdpx, pdpy, ignored

def finite_differences(x: np.ndarray, y: np.ndarray):
    """
    Use the unique x values within a leaf to compute finite differences. Given, n unique
    x values return n-1 derivative estimates. We start by grouping the leaf x,y by x
    and then collect the average y. The unique x and y averages are the new x and y pairs.

    The slope for each x is the forward discrete difference:

        (y_{i+1} - y_i) / (x_{i+1} - x_i)

    At right edge, there is no following x value so we could use backward difference:

        (y_i - y_{i-1}) / (x_{i+1} - x_i)

    But we don't use that last value as there's nothing to plot pass last value; slope
    at last value therefore is worthless.

    I experimented with the center difference using a secant method:

        (y_{i+1} - y_{i-1}) / (x_{i+1} - x_{i-1})

    which is Method A from Veldman, A.E.P., Rinzema, K. Playing with nonuniform grids.
    J Eng Math 26, 119–130 (1992).  https://doi.org/10.1007/BF00043231
    https://www.rug.nl/research/portal/files/3332271/1992JEngMathVeldman.pdf

    The np.gradient() function uses Method B from Veldman and Rinzema but that one seems
    to give much worse derivatives for sparse data.  In the end, forward difference seemed
    to get the best approximation for synthetic data, particularly sparse data. Since
    it is simplest, I went with it.

    If the ordinal/ints are exactly one unit part, then it's just y_{i+1} - y_i. If
    they are not consecutive, we do not ignore isolated x_i as it ignores too much data.
    E.g., if x is [1,3,4] and y is [9,8,10] then the x=2 coordinate is spanned as part
    of 1 to 3. The two slopes are [(8-9)/(3-1), (10-8)/(4-3)] and bin widths are [2,1].

    If there is exactly one unique x value in the leaf, the leaf provides no information
    about how X[colname] contributes to changes in y. We have to ignore this leaf.
    """
    ignored = 0

    # Group by x, take mean of all y with same x value (they come back sorted too)
    uniq_x = np.unique(x)
    avg_y = np.array([y[x==ux].mean() for ux in uniq_x])

    if len(uniq_x)==1:
        # print(f"ignore {len(x)} in discrete_xc_space")
        ignored += len(x)
        return np.array([[0]],dtype=x.dtype), np.array([0.0]), ignored

    # FORWARD DIFF
    x_deltas = np.diff(uniq_x)
    y_deltas = np.diff(avg_y)
    leaf_slopes = y_deltas / x_deltas  # "rise over run"

    # AVERAGE AROUND CENTER DIFF
    # At position i, take average of forward slope from y[i-1] to y[i] and
    # slope from y[i] to y[i+1].
    # leaf_slopes = [leaf_slopes[0]] + list((leaf_slopes[:-1] + leaf_slopes[1:]) / 2)
    # leaf_slopes = np.array(leaf_slopes)
    # dang, doesn't seem to work

    # CENTER DIFF
    # x_deltas2 = uniq_x[2:] - uniq_x[:-2]  # this is empty if |uniq_x|==2
    # y_deltas2 = avg_y[2:] - avg_y[:-2]
    # dydx0 = (avg_y[1] - avg_y[0]) / (uniq_x[1] - uniq_x[0]) # forward diff for i=0
    # leaf_slopes_ctr = [dydx0] + list(y_deltas2 / x_deltas2)  # "rise over run, stride 2"
    # leaf_slopes_ctr = np.array(leaf_slopes_ctr)

    # leaf_slopes = np.gradient(avg_y, uniq_x)[:-1] # drop last derivative as we won't use it
    # print("ctr",list(leaf_slopes_ctr))
    # print("grd",list(leaf_slopes))

    leaf_xranges = np.array(list(zip(uniq_x, uniq_x[1:])))

    return leaf_xranges, leaf_slopes, ignored

def collect_discrete_slopes(rf, X_col, X_not_col, y):
    """
    For each leaf of each tree of the decision tree or RF rf (trained on all features
    except colname), get the leaf samples then isolate the X[colname] values
    and the target y values.  Compute the y deltas between unique X[colname] values.
    Like performing piecewise linear regression of X[colname] vs y
    to get the slopes in various regions of X[colname].  We don't need to subtract
    the minimum y value before regressing because the slope won't be different.
    (We are ignoring the intercept of the regression line).

    Return for each leaf, the ranges of X[colname] partitions,
    associated slope for each range, and number of ignored samples.
    """
    # start = timer()
    leaf_slopes = []   # drop or rise between discrete x values
    leaf_xranges = []  # drop is from one discrete value to next

    ignored = 0

    leaves = leaf_samples(rf, X_not_col)
    y = y.values

    if False:
        nnodes = rf.estimators_[0].tree_.node_count
        print(f"Partitioning 'x not {colname}': {nnodes} nodes in (first) tree, "
              f"{len(rf.estimators_)} trees, {len(leaves)} total leaves")

    for samples in leaves:
        leaf_x = X_col[samples]
        # leaf_x = one_leaf_samples[]#.reshape(-1,1)
        leaf_y = y[samples]

        if np.abs(np.min(leaf_x) - np.max(leaf_x)) < 1.e-8: # faster than np.isclose()
            # print(f"ignoring xleft=xright @ {r[0]}")
            ignored += len(leaf_x)
            continue

        leaf_xranges_, leaf_slopes_, ignored_ = \
            finite_differences(leaf_x, leaf_y)

        leaf_slopes.extend(leaf_slopes_)
        leaf_xranges.extend(leaf_xranges_)
        ignored += ignored_

    if len(leaf_xranges)==0:
        # make sure empty list has same shape (jit complains)
        leaf_xranges = np.array([]).reshape(0, 0)
    else:
        leaf_xranges = np.array(leaf_xranges)
    leaf_slopes = np.array(leaf_slopes)

    # stop = timer()
    # if verbose: print(f"collect_discrete_slopes {stop - start:.3f}s")
    return leaf_xranges, leaf_slopes, ignored

# We get about 20% boost from parallel but limits use of other parallelism it seems;
# i get crashes when using multiprocessing package on top of this.
# If using n_jobs=1 all the time for importances, then turn jit=False so this
# method is not used
@jit(nopython=True, parallel=True) # use prange not range.
def avg_slopes_at_x_jit(uniq_x, leaf_ranges, leaf_slopes):
    """
    Compute the average of leaf_slopes at each uniq_x.

    Value at max(x) is NaN since we have no data beyond that point and so there is
    no forward difference. If last range is 4..5 then slope at 5 is nan since we
    don't know where it's going to go from there.
    """
    nx = uniq_x.shape[0]
    nslopes = leaf_slopes.shape[0]
    slopes = np.empty(shape=(nx, nslopes), dtype=np.double)
    for i in prange(nslopes):
        xr, slope = leaf_ranges[i], leaf_slopes[i]
        # Compute slope all the way across uniq_x but then trim line so
        # slope is only valid in range xr; don't set slope on right edge
        slopes[:, i] = np.where( (uniq_x < xr[0]) | (uniq_x >= xr[1]), np.nan, slope)

    # Slope values could be genuinely zero so we use nan not 0 for out-of-range.

    # Now average horiz across the matrix, averaging within each range
    # It's possible that some some rows would be purely NaN, indicating there are no
    # slopes for that X[colname] value. This can happen when we ignore some leaves,
    # when they have a single unique X[colname] value.

    # Compute:
    #   avg_slope_at_x = np.mean(slopes[good], axis=1)  (numba doesn't allow axis arg)
    #   slope_counts_at_x = nslopes - np.isnan(slopes).sum(axis=1)
    avg_slope_at_x = np.zeros(shape=nx)
    slope_counts_at_x = np.zeros(shape=nx)
    for i in prange(nx):
        row = slopes[i, :]
        n_nan = np.sum(np.isnan(row))
        avg_slope_at_x[i] = np.nan if n_nan==nslopes else np.nanmean(row)
        slope_counts_at_x[i] = nslopes - n_nan

    # return average slope at each unique x value and how many slopes included in avg at each x
    return avg_slope_at_x, slope_counts_at_x

# Hideous copying of avg_values_at_x_jit() to get different kinds of jit'ing. This is slower by 20%
# than other version but can run in parallel with multiprocessing package.
@jit(nopython=True)
def avg_slopes_at_x_nonparallel_jit(uniq_x, leaf_ranges, leaf_slopes):
    """
    Compute the weighted average of leaf_slopes at each uniq_x.

    Value at max(x) is NaN since we have no data beyond that point and so there is
    no forward difference.
    """
    nx = len(uniq_x)
    nslopes = len(leaf_slopes)
    slopes = np.zeros(shape=(nx, nslopes))
    for i in range(nslopes):
        xr, slope = leaf_ranges[i], leaf_slopes[i]
        # Compute slope all the way across uniq_x but then trim line so
        # slope is only valid in range xr; don't set slope on right edge
        slopes[:, i] = np.where( (uniq_x < xr[0]) | (uniq_x >= xr[1]), np.nan, slope)

    # Slope values could be genuinely zero so we use nan not 0 for out-of-range.

    # Now average horiz across the matrix, averaging within each range
    # It's possible that some some rows would be purely NaN, indicating there are no
    # slopes for that X[colname] value. This can happen when we ignore some leaves,
    # when they have a single unique X[colname] value.

    # Compute:
    #   avg_value_at_x = np.mean(slopes[good], axis=1)  (numba doesn't allow axis arg)
    #   slope_counts_at_x = nslopes - np.isnan(slopes).sum(axis=1)
    avg_value_at_x = np.zeros(shape=nx)
    slope_counts_at_x = np.zeros(shape=nx)
    for i in range(nx):
        row = slopes[i, :]
        n_nan = np.sum(np.isnan(row))
        avg_value_at_x[i] = np.nan if n_nan==nslopes else np.nanmean(row)
        slope_counts_at_x[i] = nslopes - n_nan

    # return average slope at each unique x value and how many slopes included in avg at each x
    return avg_value_at_x, slope_counts_at_x

def catwise_leaves(rf, X_not_col, X_col, y, max_catcode):
    """
    Return a 2D array with the average y value for each category in each leaf.
    Choose the cat code of smallest avg y as the reference category. I used to think it
    was arbitrary but now I realize that we have to mimic what StratPD does for
    numerical variables. By integrating the y deltas, we get back exactly the y
    values minus the minimum y. E.g., for x=[1,2,3] and y=[7,10,15], StratPD gets
    dx = [1,1] and dy=[3,5] then cumsum of dy gets [3,8] and we stick 0 on front:
    pdpy = [0,3,8].  To mimic the same thing with CatStratPD, we simply subtract the
    minimum for all categories in this leaf. If cats=[1,2,3] and y=[7,10,15], then
    we subtract 7 from y = [0,3,8] and we arrive at the same answer. For situations
    where the numerical var PD never skips over a valid x (extrapolating slope), it
    should give the same answer as categorical var PD. Subtracting the min y also
    has the benefit that debugging is easier because, within a leaf, all deltas are
    positive. During merging, we can still get negative deltas as we lineup cats/deltas,
    but those are true values. We should not shift everything to be positive for
    the final merged pdpy.

    Normalize the y values into deltas by subtracting the avg y value for the
    reference category from the avg y for all categories. The reference category is
    the category of the minimum avg y.

    The columns are the y avg value changes per cat found in a single leaf as
    they differ from the reference cat y average. Each row represents a category level. E.g.,

    row
    cat           leaf0       leaf1
     0       166.430176  186.796956
     1       219.590349  176.448626

    Cats are possibly noncontiguous with nan rows for cat codes not present. Not all
    values in a leaf column will be non-nan.  Only those categories mentioned in
    a leaf have values.  Shape is (max cat + 1, num leaves).

    Within a single leaf, there will typically only be a few categories represented.
    """
    leaves = leaf_samples(rf, X_not_col)

    leaf_deltas = np.full(shape=(max_catcode+1, len(leaves)), fill_value=np.nan)
    leaf_counts = np.zeros(shape=(max_catcode+1, len(leaves)), dtype=int)
    keep_leaf_idxs = np.full(shape=(len(leaves),), fill_value=True, dtype=bool)

    ignored = 0
    for leaf_i in range(len(leaves)):
        sample = leaves[leaf_i]
        leaf_cats = X_col[sample]
        leaf_y = y[sample]
        # perform a groupby(catname).mean()
        uniq_leaf_cats, count_leaf_cats = np.unique(leaf_cats, return_counts=True) # comes back sorted
        avg_y_per_cat = np.array([leaf_y[leaf_cats==cat].mean() for cat in uniq_leaf_cats])
        # print("uniq_leaf_cats",uniq_leaf_cats,"count_y_per_cat",count_leaf_cats)

        # if len(uniq_leaf_cats)==1 then we have single cat avg y and its delta is 0
        # but keep it as we'll treat as isolated group later and add marginal y for
        # this cat to get partial dependence value.

        # Can use any cat code as refcat; same "shape" of delta vec regardless of which we
        # pick. The vector is shifted/up or down but cat y's all still have the same relative
        # delta y. Might as well just pick the cat of smallest avg y.
        # Previously, I picked a random# reference category but that is unnecessary.
        # We will shift this vector during the merge operation so which we pick
        # here doesn't matter.
        idx_of_ref_cat_in_leaf = np.nanargmin(avg_y_per_cat)
        delta_y_per_cat = avg_y_per_cat - avg_y_per_cat[idx_of_ref_cat_in_leaf]
        # print("delta_y_per_cat",delta_y_per_cat)

        # Store into leaf i vector just those deltas we have data for
        # leave cats w/o representation as nan (uses index to figure out which rows to alter)
        leaf_deltas[uniq_leaf_cats, leaf_i] = delta_y_per_cat
        leaf_counts[uniq_leaf_cats, leaf_i] = count_leaf_cats

    # See unit test test_catwise_leaves:test_two_leaves_with_2nd_ignored()
    leaf_deltas = leaf_deltas[:,keep_leaf_idxs]
    leaf_counts = leaf_counts[:,keep_leaf_idxs]
    return leaf_deltas, leaf_counts, ignored

def cat_partial_dependence(X, y,
                           colname,  # X[colname] expected to be numeric codes
                           max_catcode=None,  # if we're bootstrapping, might see diff max's so normalize to one max
                           n_trees=1,
                           min_samples_leaf=5,
                           max_features=1.0,
                           rf_bootstrap=False,
                           supervised=True,
                           verbose=False):
    X_not_col = X.drop(colname, axis=1).values
    X_col = X[colname].values
    if (X_col<0).any():
        raise ValueError(f"Category codes must be > 0 in column {colname}")
    if not np.issubdtype(X_col.dtype, np.integer):
        raise ValueError(f"Category codes must be integers in column {colname} but is {X_col.dtype}")
    if max_catcode is None:
        max_catcode = np.max(X_col)
    if supervised:
        rf = RandomForestRegressor(n_estimators=n_trees,
                                   min_samples_leaf=min_samples_leaf,
                                   bootstrap = rf_bootstrap,
                                   max_features = max_features,
                                   oob_score=False)
    else:
        print("USING UNSUPERVISED MODE")
        X_synth, y_synth = conjure_twoclass(X)
        rf = RandomForestClassifier(n_estimators=n_trees,
                                    min_samples_leaf=min_samples_leaf,  # * 2, # there are 2x as many samples (X,X') so must double leaf size
                                    bootstrap=rf_bootstrap,
                                    max_features=max_features,
                                    oob_score=False)
        rf.fit(X_synth.drop(colname,axis=1), y_synth)

    rf.fit(X_not_col, y)
    if verbose and supervised:
        print(f"CatStrat Partition RF: dropping {colname} training R^2 {rf.score(X_not_col, y):.2f}")

    leaf_deltas, leaf_counts, ignored = \
        catwise_leaves(rf, X_not_col, X_col, y.values, max_catcode)

    uniq_x = np.unique(X_col)
    # Ignoring other vars, what is average y for all records with same catcode?
    # We need if avg_values_at_cat finds disjoint sets
    marginal_avg_y_per_cat = np.full(shape=(max_catcode+1,), fill_value=np.nan)
    for cat in uniq_x:
        marginal_avg_y_per_cat[cat] = y[X_col == cat].mean()

    avg_per_cat, count_per_cat = \
        avg_values_at_cat(leaf_deltas, leaf_counts, marginal_avg_y_per_cat, verbose=verbose)

    if verbose:
        print(f"CatStratPD Num samples ignored {ignored} for {colname}")

    return leaf_deltas, leaf_counts, avg_per_cat, count_per_cat, ignored

def avg_values_at_cat(leaf_deltas, leaf_counts,
                      marginal_avg_y_per_cat=None, # only used if disjoint cat sets exist
                      max_iter=3, verbose=False):
    """
    Merge the deltas for categories from all of the leaves. If there
    is at least one category in common, then to groups of deltas can
    be merged. It's possible, however, that there are multiple
    disjoint groups of categories with no cats in common. Previously I
    would just ignore those, but now I realize there could be
    important groups of categories that don't get included. For
    example with the bulldozer data set it was common to see the vast
    majority of categories being ignored. Obviously this dramatically
    underestimates the impact of a categorical variable.

    We don't have categories in common between disjoint groups but we
    can still estimate their relative position, otherwise we'd be
    forced to start all disjoint groups at zero. If we treat each
    group as a meta-category, then we can compare the average y for
    categories in a group. For meta-cats A, B, C with avg y =
    [100,130,40] then the meta-deltas are [0,30,-60].  The base for
    the zeros in A, B, C will be 0, 30, -60 so we just add that to all
    cat deltas in A, B, C.  Note: the y used for mean is not the leaf
    deltas but y from training set.

    This is effectively the marginal delta between those meta-groups,
    but the fact that they are disjoint means that there were no
    situations where, all else being equal, the associated leaves had
    categories in common. While the overall variable might be highly
    codependent, such as ModelID (with YearMade, MachineHours,...) on
    bulldozer price, the individual groups WITHIN a variable could be
    totally independent. Imagine typical construction bulldozers
    versus the giant earthmovers used in mining.  It's extremely
    unlikely that stratification would yield groups containing both
    construction bulldozers and mining bulldozers. That hints that
    the average difference in price between the two groups would not be
    biased (too much). Besides, it's the best we can do given we
    literally have no information about the relative cost of a mining
    bulldozer and a construction bulldozer. All we can do is compare
    the average price of a bulldozer in both categories, and use that
    to shift the two meta-cats relative to each other.

    Rather than a complicated data structure and algorithm to find all
    disjoint subsets of intersecting categories, it's easier to use the
    existing algorithm I had already that merged leaves with categories in
    common.  After merging all possible, there remains a set of leaf
    indexes indicating what leaves remain to process. So, just call function
    avg_values_at_cat_one_disjoint_region() repeatedly until the work
    set comes back as empty. That indicates all leaves have been
    integrated. The count per category vectors can simply be added
    together to get the final count per category. The catavg is the
    running vector average computed in each pass over the leaves. For
    that, we can add them vectors together to get a single vector
    (like a set union), but we still need to keep track of the different
    disjoint category groups. We can do this by collecting a list
    of cat sets.  Then we compute the average y for those categories
    for each group. Then update the catavg vector values for each
    group.

    See comment for avg_values_at_cat_one_disjoint_region()
    """
    # catavg is the running average vector and starts out as the first column (1st leaf's deltas)
    catavg = np.full(shape=(leaf_deltas.shape[0],), fill_value=np.nan, dtype=np.float)
    catgroups = []
    count_per_cat = np.zeros(shape=(leaf_deltas.shape[0],), dtype=np.int)
    work = range(0, leaf_deltas.shape[1])  # all leaf indexes added to work list
    # print("START work", work)
    while len(work)>0:
        catavg_, count_per_cat_, work = \
            avg_values_at_cat_one_disjoint_region(work, leaf_counts, leaf_deltas, max_iter, verbose)
        # print("catavg", catavg_)
        # print("count_per_cat", count_per_cat_)
        # print("remaining work",work)
        catgroup = np.where(count_per_cat_>0)[0]
        # print("catgroup", catgroup)
        catgroups.append(catgroup)
        catavg[catgroup] = catavg_[catgroup] # merge disjoint avgs for catgroup into running vector
        count_per_cat += count_per_cat_

    if len(catgroups)>1: # more than one disjoint set of categories?
        avgs = []
        for cats in catgroups:
            m = np.mean(marginal_avg_y_per_cat[cats])
            # print("cats",cats,"mean",m)
            avgs.append(m)
        # Do what StratPD does: all values are relative to left edge
        relative_group_y = np.array(avgs) - avgs[0]
        # print("group avgs", avgs)
        # print("relative_group_y", relative_group_y)
        for cats,group_deltay in zip(catgroups, relative_group_y):
            catavg[cats] += group_deltay
        # print("adjusted catavg", catavg)

    # if verbose: print("final cat avgs", parray3(catavg))
    return catavg, count_per_cat

def avg_values_at_cat_one_disjoint_region(work, leaf_counts, leaf_deltas, max_iter, verbose):
    """
    In leaf_deltas, we have information from the leaves indicating how
    much above or below each category was from the reference category
    of that leaf.  The reference category was arbitrarily selected but
    now I realize it must be the cat code associated with the minimum
    avg cat y.  All deltas within a leaf are therefore positive values
    and the relative y for the reference category is 0.  Categories
    not mentioned in the leaf, will have NAN values.

    The goal is to merge all of the columns in leaf_deltas, despite
    the fact that they do not have the same reference category. We
    init a running average vector to be the first column of category
    deltas. Then we attempt to merge each of the other columns into
    the running average. We make multiple passes over the columns of
    leaf_deltas until nothing changes, we hit the maximum number of
    iterations, or everything has merged.

    To merge vector v (column j of leaf_deltas) into catavg, select a
    category, index ix, in common at random.  Subtract v[ix] from v so
    that ix is v's new reference and v[ix]=0. Add catavg[ix] to the
    adjusted v so that v is now comparable to catavg. We can now do a
    weighted average of catavg and v, paying careful attention of NaN.

    Previously, I was picking a random category for merging in an
    effort to reduce the effect of outliers, with the assumption that
    outliers were rare. Given 5 categories in common between the
    running average vector and a new vector, randomly picking one
    means a 1/5 chance of picking the outlier.  Outliers as reference
    categories shift the outlierness to all other categories. Boooo

    Now, I select the category in common that has the most evidence:
    the category associated with the most number of observations.

    It's possible that more than a single value within a leaf_deltas
    vector is 0.  I.e., the reference category value is always 0 in
    the vector, but there might be another category whose value was
    the same y, giving a 0 relative value.

    Example:

    leaf_deltas

    [[ 0. nan nan]
     [ 1. nan nan]
     [nan  0. nan]
     [nan  3.  0.]
     [nan  3.  2.]
     [ 4. nan nan]
     [ 5. nan nan]]

    leaf_counts

    [[1 0 0]
     [1 0 0]
     [0 1 0]
     [0 1 1]
     [0 1 1]
     [1 0 0]
     [1 0 0]]
    """
    initial_leaf_idx = work[0]
    work = set(work[1:])
    catavg = leaf_deltas[:, initial_leaf_idx]  # init with first ref category (column)
    catavg_weight = leaf_counts[:, initial_leaf_idx]
    completed = {-1}  # init to any nonempty set to enter loop
    iteration = 1
    # Three passes should be sufficient to merge all possible vectors, but
    # I'm being paranoid here and allowing it to run until completion or some maximum iterations
    while len(work) > 0 and len(completed) > 0 and iteration <= max_iter:
        # print(f"PASS {iteration} len(work)", len(work))
        completed = set()
        for j in work:  # for remaining leaf index in work list, avg in the vectors
            v = leaf_deltas[:, j]
            are_intersecting = ~np.isnan(catavg) & ~np.isnan(v)
            intersection_idx = np.where(are_intersecting)[0]

            # print(intersection_idx)
            if len(intersection_idx) == 0:  # found something to merge into catavg?
                continue

            # Merge column j into catavg vector
            # cat for merging is the one with most supporting evidence
            cur_weight = leaf_counts[:, j]

            PICK_MOST_CONF = False
            if PICK_MOST_CONF:
                ix = np.argmax(np.where((cur_weight > 0) & are_intersecting, cur_weight, 0))
                shifted_v = v - v[ix]  # make ix the reference cat in common
                relative_to_value = catavg[ix]  # corresponding value in catavg
                adjusted_v = shifted_v + relative_to_value  # adjust so v is mergeable with catavg
            else:
                adjusted_v = compute_avg_merge_candidate(catavg, v, intersection_idx)

            prev_catavg = catavg  # track only for verbose/debugging purposes
            catavg = nanavg_vectors(catavg, adjusted_v, catavg_weight, cur_weight)
            # Update weight of running avg to incorporate "mass" from v
            catavg_weight += cur_weight
            if verbose:
                # TODO: broken
                print(f"{ix:-2d} : vec to add =", parray(v), f"- {v[ix]:.2f}")
                print("     shifted    =", parray(shifted_v),
                      f"+ {relative_to_value:.2f}")
                print("     adjusted   =", parray(adjusted_v), "*", cur_weight)
                print("     prev avg   =", parray(prev_catavg), "*",
                      catavg_weight - cur_weight)
                print("     new avg    =", parray(catavg))
                print()
            completed.add(j)
        iteration += 1
        work = work - completed
    return catavg, catavg_weight, list(work)

def compute_avg_merge_candidate(catavg, v, intersection_idx):
    """
    Given intersecting deltas in catavg and v, compute average delta
    one could merge into running average.  If one cat is an outlier,
    picking that really distorts the vector we merge into running
    average vector. So, effectively merge using all as the ref
    cat in common by merging in average of all possible refcats.

    When there is no noise in y, the average merge candidate is
    the same as any single candidate. So, with no noise, we get
    exact answer; averaging here doesn't cost us anything. It
    only helps to spread noise across categories.
    """
    merge_candidates = []
    for i in intersection_idx:
        merge_candidates.append(v - v[i] + catavg[i])
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", category=RuntimeWarning)
        # We get "Mean of empty slice" when all entries are Nan but we want that.
        v = np.nanmean(merge_candidates, axis=0)
    return v

# -------------- S U P P O R T ---------------

def scramble(X : np.ndarray) -> np.ndarray:
    """
    From Breiman: https://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm
    "...the first coordinate is sampled from the N values {x(1,n)}. The second
    coordinate is sampled independently from the N values {x(2,n)}, and so forth."
    """
    X_rand = X.copy()
    ncols = X.shape[1]
    for col in range(ncols):
        X_rand[:,col] = np.random.choice(X[:,col], len(X), replace=True)
    return X_rand

def df_scramble(X : pd.DataFrame) -> pd.DataFrame:
    """
    From Breiman: https://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm
    "...the first coordinate is sampled from the N values {x(1,n)}. The second
    coordinate is sampled independently from the N values {x(2,n)}, and so forth."
    """
    X_rand = X.copy()
    for colname in X:
        # X_rand[colname] = np.random.choice(X[colname], len(X), replace=True)
        X_rand[colname] = X_rand[colname].sample(frac=1.0)
    return X_rand

def conjure_twoclass(X):
    """
    Make new data set 2x as big with X and scrambled version of it that
    destroys structure between features. Old is class 0, scrambled is class 1.
    """
    if isinstance(X, pd.DataFrame):
        X_rand = df_scramble(X)
        X_synth = pd.concat([X, X_rand], axis=0)
    else:
        X_rand = scramble(X)
        X_synth = np.concatenate([X, X_rand], axis=0)
    y_synth = np.concatenate([np.zeros(len(X)),
                              np.ones(len(X_rand))], axis=0)
    return X_synth, pd.Series(y_synth)

def nanavg_vectors(a, b, wa=1.0, wb=1.0):
    """
    Add two vectors a+b but support nan+x==x and nan+nan=nan
    np.nanmean works to get nan+nan=nan, but for weighted avg
    we need to divide by wa+wb after using nansum. nansum gives
    0 not nan it seems when adding nan+nan. Do it the hard way.
    """
    a_nan = np.isnan(a)
    b_nan = np.isnan(b)
    c = a*wa + b*wb               # weighted average where both are non-nan
    c /= zero_as_one(wa+wb)       # weighted avg
    c[a_nan] = b[a_nan]           # copy any stuff where b has only value (unweighted into result)
    in_a_not_b = (~a_nan) & b_nan
    c[in_a_not_b] = a[in_a_not_b] # copy stuff where a has only value
    return c

def nanmerge_matrix_cols(A):
    """
    Add all vertical vectors in A but support nan+x==x and nan+nan=nan.
    """
    s = np.nansum(A, axis=1)
    all_nan_entries = np.isnan(A)
    # if all entries for a cat are nan, make sure sum s is nan for that cat
    s[all_nan_entries.all(axis=1)] = np.nan
    return s

def zero_as_one(a):
    return np.where(a == 0, 1, a)

def parray(a):
    if type(a[0])==np.int64:
        return '[ ' + (' '.join([f"{x:6d}" for x in a])).strip() + ' ]'
    else:
        return '[ ' + (' '.join([f"{x:6.2f}" for x in a])).strip() + ' ]'

def parray3(a):
    return '[ ' + (' '.join([f"{x:6.3f}" for x in a])).strip() + ' ]'


import sys
import os
from tkinter import *
import threading
import time
from h5_listener import h5_listener
from receipt_printer import receipt_printer
import fasteners
import selenium.webdriver as webdriver
from selenium.common.exceptions import NoSuchElementException

# 数据读写锁
data_lock = threading.Lock()

# 线程
threads = []

# 读取配置文件
def readConfig():
    if not os.path.exists('./config.ini'):
        sys.exit(0)  # 结束程序
    config = {}
    # 读取config文件，获取模板的每个输入框的对应方式
    with open('./config.ini', encoding='utf-8') as f:
        lines = f.readlines()
        for line in lines:
            # print(line)
            if line[0] == "#":
                continue
            if '=' in line:
                data = line.strip().split('=')
                config[data[0]] = data[1]
    print(config)
    return config

# 启动子线程
def start_threads():
    with data_lock:
        shared_data['running'] = True
        shared_data['frequency'] = frequency
    global threads
    threads = []
    threads.append(threading.Thread(target=h5_listener, args=(shared_data, data_lock)))
    threads.append(threading.Thread(target=receipt_printer, args=(shared_data, data_lock)))

    for thread in threads:
        thread.start()

# 暂停子线程
def stop_threads():
    with data_lock:
        shared_data['running'] = False
        shared_data['frequency'] = 0.01

# 更新GUI
def update_gui(status_var, url_var, current_code_var, print_status_var, previous_code_var):
    status_var.set('Status: Running' if shared_data['running'] else 'Status: Stopped')
    url_var.set('Current Url: ' + shared_data['url'])
    current_code_var.set('Current Take Code: ' + shared_data['current_code'])
    print_status_var.set('Print Status: Printing' if shared_data['print_flag'] else 'Print Status: Printed')
    previous_code_var.set('Previous Take Code: ' + shared_data['previous_code'])
    root.after(1000, update_gui, status_var, url_var, current_code_var, print_status_var, previous_code_var)  # 每秒更新一次

# 当窗口关闭时
def on_close():

    try:
        root.destroy()  # 销毁窗口

        # 关闭浏览器driver
        with data_lock:
            driver = shared_data['driver']

        driver.quit()

        stop_threads()  # 停止所有线程

        # 等待所有子线程完成
        for thread in threads:
            thread.join()

        # pid_lock.release()  # 释放锁
        sys.exit(0)  # 结束程序

    except Exception as e:
            pass

# def restart():

#     with data_lock:
#         if shared_data['restart_flag']:
#             previous_driver = shared_data['driver']
#             # 关闭浏览器
#             try:
#                 # 尝试关闭
#                 previous_driver.quit()
#             except WebDriverException:
#                 print("Browser has been closed")
#             # 创建一个Chrome浏览器实例
#             new_driver = webdriver.Chrome()
#             shared_data['driver'] = new_driver
#             # 打开一个网页
#             new_driver.get(h5_url)
#             # 以全屏模式打开Chrome浏览器
#             new_driver.fullscreen_window()
#         else:
#             shared_data['restart_flag'] = True

# 读取配置文件

config = readConfig()

h5_url = ''
frequency = 2
zoom = 1.3
restart_flag = False

# 读取监听url
if config['url'] == '':
    sys.exit(0)  # 结束程序
else:
    h5_url = config['url']
    print(config['url'])

if config['time'] != '':
    frequency = config['time']

if config['zoom'] != '':
    try:
        zoom =config['zoom']
    except Exception:
        pass

# 创建一个Chrome浏览器实例
chrome_options = webdriver.ChromeOptions()

argument_str = "--force-device-scale-factor=" + str(zoom)
chrome_options.add_argument(argument_str)

driver = webdriver.Chrome(options=chrome_options)
# 打开一个网页
driver.get(h5_url)
# 以全屏模式打开Chrome浏览器
driver.fullscreen_window()

# 全局共享变量
shared_data = {
    'driver': driver,
    'url': h5_url,
    'zoom': zoom,
    'current_code': '',
    'previous_code': '',
    'print_flag': False,
    'running' : True,
    'frequency' : frequency,
    'restart_flag' : restart_flag,
    'receipt' : {},
    'new_code_event': threading.Event()
}

# 限制同时只能运行一个程序，禁止多开
# 利用文件锁技术，事先选定一个文件作为锁，运行第一个进程时锁住此文件
# 再次运行程序时检测文件此是否被锁定，若被锁定就说明已有一个程序正在运行
# pid_lock = fasteners.InterProcessLock('./H5Printer.lock')
# gotten = pid_lock.acquire(blocking=False)
# if not gotten:
#     print("Another instance of this application currently running.")
#     sys.exit(1)

# GUI
root = Tk()
root.geometry('700x350')  # 设置窗口的宽为300，高为200
root.minsize(350, 250)  # 设置窗口的最小宽度为300，最小高度为200
root.title('H5 Printer')  # 设置窗口的标题
root.protocol("WM_DELETE_WINDOW", on_close)  # 当窗口关闭时调用on_close函数

# 设置文本
status_var = StringVar()
url_var = StringVar()
current_code_var = StringVar()
previous_code_var = StringVar()
print_status_var = StringVar()

Button(root,
       text="Stop/Start",
       command=lambda: start_threads() if not shared_data['running'] else stop_threads(),
       bg="lightblue",
       fg="orange",
       padx=10,
       pady=10,
       font=("Arial", 20, "bold")
       ).pack()
Label(root,
      textvariable=status_var,
      font=("Arial", 20, "bold"),
      fg="red",
      ).pack()
Label(root,
      textvariable=url_var,
      font=("Arial", 15)
      ).pack()
Label(root,
      textvariable=current_code_var,
      font=("Arial", 15, "bold")
      ).pack()
Label(root,
      textvariable=print_status_var,
      font=("Arial", 15)
      ).pack()
Label(root,
      textvariable=previous_code_var,
      font=("Arial", 15)
      ).pack()
# Button(root,
#        text="Restart H5",
#        command=restart(),
#        bg="lightblue",
#        fg="orange",
#        padx=10,
#        pady=10,
#        font=("Arial", 15, "bold")
#        ).pack()

update_gui(status_var, url_var, current_code_var, print_status_var, previous_code_var)

# 自动开始监听
start_threads()

# 最小化窗口
root.iconify()

root.mainloop()