"""Define Math class.

Classes
-------
Math
    Configures Math frame.
"""
import tkinter as tk
from tkinter import ttk
from .toolbar_inner_frames import Fraction, Matrix, OperatorsMenu, Brackets, SaveEquation
from functools import partial
from files import saved_equations

class Math(ttk.Frame):
    """A class configuring Math frame; also creates Brackets and OperatorsMenu menus.

    Attributes
    ----------
    controller : widget object
        a reference to master widget object to provide access to its attributes/methods
    frac : tk.PhotoImage(file)
        a reference to Image object so image is not garbage collected
    integral : tk.PhotoImage(file)
        a reference to Image object so image is not garbage collected
    cyclic_integral : tk.PhotoImage(file)
        a reference to Image object so image is not garbage collected
    binomial : tk.PhotoImage(file)
        a reference to Image object so image is not garbage collected
    sum : tk.PhotoImage(file)
        a reference to Image object so image is not garbage collected
    product : tk.PhotoImage(file)
        a reference to Image object so image is not garbage collected
    coproduct : tk.PhotoImage(file)
        a reference to Image object so image is not garbage collected
    limit : tk.PhotoImage(file)
        a reference to Image object so image is not garbage collected
    fraction : Fraction()
        a Fraction toplevel window
    matrix : Matrix()
        a Matrix toplevel window
    save_equation : SaveEquation()
        a SaveEquation toplevel window

    Methods
    -------
    insert_amsmath()
        Inserts LaTeX command for amsmath package in currently visible scrolled text box
    insert_inline()
        Inserts LaTeX commands for inline mode in currently visible scrolled text box
    insert_display()
        Inserts LaTeX commands for display mode in currently visible scrolled text box
    superscript()
        Inserts LaTeX command for supersript in currently visible scrolled text box
    subscript()
        Inserts LaTeX command for subscript in currently visible scrolled text box
    insert_frac()
        Inserts LaTeX command for fraction in currently visible scrolled text box
    launch_fraction()
        Creates Fraction toplevel window
    insert_binomial()
        Inserts LaTeX command for binomial in currently visible scrolled text box
    launch_matrix()
        Creates Matrix toplevel window
    insert_integral()
        Inserts LaTeX command for integral in currently visible scrolled text box
    insert_cylic_integral()
        Inserts LaTeX command for cylic integral in currently visible scrolled text box
    insert_sum()
        Inserts LaTeX command for sum in currently visible scrolled text box
    insert_product()
        Inserts LaTeX command for product in currently visible scrolled text box
    insert_coproduct()
        Inserts LaTeX command for coproduct in currently visible scrolled text box
    insert_limit()
        Inserts LaTeX command for limit in currently visible scrolled text box
    create_equation(numbered)
        Inserts LaTeX commands for equation environment in currently visible scrolled text box
    label()
        Inserts LaTeX command for label in currently visible scrolled text box
    align()
        Inserts LaTeX commands for align environment in currently visible scrolled text box
    gather()
        Inserts LaTeX commands for gather environment in currently visible scrolled text box
    multiline()
        Inserts LaTeX commands for multiline environment in currently visible scrolled text box
    launch_save_equation()
        Creates SaveEquation toplevel window
    load_saved_equations()
        Reads saved equations from saved_equations.py and adds as commands to saved_equations_button_menu
    insert_equation(equation)
        Inserts value of equation in currently visible scrolled text box
    delete_equation(equation)
        Deletes equation from saved_equations.py and saved_equations_button_menu
    """

    def __init__(self, container, controller, **kwargs):
        """Initialises Math frame.

        Takes container and **kwargs and passes them to parent ttk.Frame class constructor. Initialises ttk.Frame
        object. Creates necessary attributes. Creates toolbar buttons, and Brackets and OperatorsMenu menus.

        Parameters
        ----------
        self
            Math object
        container : widget object
            Master widget
        controller : widget object
            A reference to master widget object to provide access to its attributes/methods
        **kwargs
            Arbitrary number of key word arguments
        """
        super().__init__(container, **kwargs)

        # Create reference to Toolbar object to provide access to its attributes/methods
        self.controller = controller

        # Create references to images, so they aren't garbage collected. To be used as icons on buttons
        self.frac = tk.PhotoImage(file=r"assets/fraction.png")
        self.integral = tk.PhotoImage(file=r"assets/integral.png")
        self.cyclic_integral = tk.PhotoImage(file=r"assets/cyclic_integral.png")
        self.binomial = tk.PhotoImage(file=r"assets/binomial.png")
        self.sum = tk.PhotoImage(file=r"assets/sum.png")
        self.product = tk.PhotoImage(file=r"assets/product.png")
        self.coproduct = tk.PhotoImage(file=r"assets/coproduct.png")
        self.limit = tk.PhotoImage(file=r"assets/limit.png")

        # Make row 0 fill all available space
        self.rowconfigure(0, weight=1)

        # Create toolbar buttons
        amsmath_button = ttk.Button(
            self,
            text="Use amsmath Package",
            command=self.insert_amsmath
        )

        inline_button = ttk.Button(
            self,
            text="Inline Mode",
            command=self.insert_inline
        )

        display_button = ttk.Button(
            self,
            text="Display Mode",
            command=self.insert_display
        )

        brackets_button = ttk.Menubutton(
            self,
            text="Brackets",
            width=8
        )
        # Create Brackets menu and associate with brackets menubutton
        brackets_button_menu = Brackets(brackets_button, self.controller, tearoff=False)
        brackets_button.config(menu=brackets_button_menu)

        superscript = ttk.Button(
            self,
            text="x\N{SUPERSCRIPT TWO}",
            command=self.superscript,
            padding=(-18, 0)
        )

        subscript = ttk.Button(
            self,
            text="x\N{SUBSCRIPT TWO}",
            command=self.subscript,
            padding=(-18, 0)
        )

        frac = ttk.Button(
            self,
            image=self.frac,
            command=self.insert_frac,
            padding=(0, 0)
        )

        self.fraction = None

        launch_fraction = ttk.Button(
            self,
            text="Fraction",
            command=self.launch_fraction
        )

        binomial = ttk.Button(
            self,
            image=self.binomial,
            command=self.insert_binomial,
            padding=(0, 0)
        )

        self.matrix = None

        matrix_button = ttk.Button(
            self,
            text="Matrix",
            command=self.launch_matrix
        )

        integral = ttk.Button(
            self,
            image=self.integral,
            command=self.insert_integral,
            padding=(0, 0)
        )

        cylic_integral = ttk.Button(
            self,
            image=self.cyclic_integral,
            command=self.insert_cyclic_integral,
            padding=(0, 0)
        )

        sum = ttk.Button(
            self,
            image=self.sum,
            command=self.insert_sum,
            padding=(0, 0)
        )

        product = ttk.Button(
            self,
            image=self.product,
            command=self.insert_product,
            padding=(0, 0)
        )

        coproduct = ttk.Button(
            self,
            image=self.coproduct,
            command=self.insert_coproduct,
            padding=(0, 0)
        )

        limit = ttk.Button(
            self,
            image=self.limit,
            command=self.insert_limit,
            padding=(0, 0)
        )

        equation_button = ttk.Menubutton(
            self,
            text="Create Equation"
        )
        # Create equation menu and associate with equation menubutton
        equation_button_menu = tk.Menu(equation_button, tearoff=False)
        equation_button.config(menu=equation_button_menu)
        # Add commands to equation menu
        numbered_p = partial(self.create_equation, True)
        equation_button_menu.add_command(
            label="Numbered Equation",
            command=numbered_p
        )
        unnumbered_p = partial(self.create_equation, False)
        equation_button_menu.add_command(
            label="Unnumbered Equation",
            command=unnumbered_p
        )
        equation_button_menu.add_command(
            label="Label",
            command=self.label
        )
        equation_button_menu.add_command(
            label="Align",
            command=self.align
        )
        equation_button_menu.add_command(
            label="Gather",
            command=self.gather
        )
        equation_button_menu.add_command(
            label="Multiline",
            command=self.multiline
        )

        self.save_equation = None

        save_equation = ttk.Button(
            self,
            text="Save Equation",
            command=self.launch_save_equation
        )

        saved_equations_button = ttk.Menubutton(
            self,
            text="Saved Equations"
        )
        # Create saved equations menu and associate with saved equations menubutton
        self.saved_equations_button_menu = tk.Menu(saved_equations_button, tearoff=False)
        saved_equations_button.config(menu=self.saved_equations_button_menu)

        # Populate saved equations menu
        self.load_saved_equations()

        operators_button = ttk.Menubutton(
            self,
            text="All Operators"
        )
        # Create OperatorsMenu menu and associate with section menubutton
        operators_button_menu = OperatorsMenu(operators_button, self.controller, tearoff=False)
        operators_button.config(menu=operators_button_menu)

        # Grid widgets in Math
        count = 0
        for widget in self.winfo_children():
            widget.grid(row=0, column=count, sticky="nesw")
            count += 1

    def insert_amsmath(self):
        """Inserts LaTeX command for amsmath package in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for amsmath package at second line of text box.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert("2.0", "\n\\usepackage{amsmath}\n")

    def insert_inline(self):
        """Inserts LaTeX commands for inline mode in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for inline mode at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\( \\)")

    def insert_display(self):
        """Inserts LaTeX commands for display mode in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for display mode at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\[ \\]")

    def superscript(self):
        """Inserts LaTeX command for superscript in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for superscript at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "^")

    def subscript(self):
        """Inserts LaTeX command for subscript in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for subscript at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "_")

    def insert_frac(self):
        """Inserts LaTeX command for fraction in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for fraction at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\frac{num}{denom}")

    def launch_fraction(self):
        """If fraction exists destroys it. Creates Fraction toplevel window named fraction."""
        if self.fraction:
            self.fraction.close()
        self.fraction = Fraction(self.controller)

    def insert_binomial(self):
        """Inserts LaTeX command for binomial in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for binomial at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\binom{num}{denom}")

    def launch_matrix(self):
        """If matrix exists destroys it. Creates Matrix toplevel window named matrix."""
        if self.matrix:
            self.matrix.close()
        self.matrix = Matrix(self.controller)

    def insert_integral(self):
        """Inserts LaTeX command for integral in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for integral at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\int_{lower}^{upper}")

    def insert_cyclic_integral(self):
        """Inserts LaTeX command for cylic integral in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for cylic integral at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\oint_{lower}")

    def insert_sum(self):
        """Inserts LaTeX command for sum in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for sum at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\sum_{lower}^{upper}")

    def insert_product(self):
        """Inserts LaTeX command for product in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for product at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\prod_{lower}^{upper}")

    def insert_coproduct(self):
        """Inserts LaTeX command for coproduct in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for coproduct at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\coprod_{lower}^{upper}")

    def insert_limit(self):
        """Inserts LaTeX command for limit in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for limit at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\lim_{lower}")

    def create_equation(self, numbered):
        """Inserts LaTeX commands for numbered or unnumbered equation in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. If numbered
        = True inserts LaTeX code for numbered section at current cursor position. Else inserts code for unnumbered
        equation. Value of numbered determined by which menu command selected.

        Parameters
        ----------
        numbered : bool
            value determining if numbered or unnumbered equation selected
        """
        current_editor = self.controller.check_current_editor()
        if numbered:
            insert = "\\begin{equation}\n\n\\end{equation}\n"
        else:
            insert = "\\begin{equation*}\n\n\\end{equation*}\n"
        current_editor.insert(tk.INSERT, insert)

    def label(self):
        """Inserts LaTeX command for label in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for label at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\label{}")

    def align(self):
        """Inserts LaTeX commands for align environment in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for align environment at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\begin{align*}\n\n\\end{align*}\n")

    def gather(self):
        """Inserts LaTeX command for gather environment in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for gather environment at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\begin{gather*}\n\n\\end{gather*}\n")

    def multiline(self):
        """Inserts LaTeX command for multiline environment in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts LaTeX
        code for multiline environment at current cursor position.
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, "\\begin{multiline*}\n\n\\end{multiline*}\n")

    def launch_save_equation(self):
        """If save_equation exists destroys it. Creates SaveEquation toplevel window named save_equation."""
        if self.save_equation:
            self.save_equation.close()
        self.save_equation = SaveEquation(self)

    def load_saved_equations(self):
        """Reads saved equations from saved_equations.py and adds as commands to saved_equations_button_menu

        For each key:value pair in the equations dictionary inside the file files/saved_equations.py, adds a menu with
        commands to insert the equation code into the currently visible editor or delete the key:value pair from the
        dictionary
        """
        try:
            for name, equation in saved_equations.equations.items():
                # Create menu inside saved equations menu for name:equation entry in dictionary
                equation_sub_menu = tk.Menu(self.saved_equations_button_menu, tearoff=0)
                self.saved_equations_button_menu.add_cascade(
                    label=name,
                    menu=equation_sub_menu
                )

                equation_p = partial(self.insert_equation, equation)
                equation_delete_p = partial(self.delete_equation, name)
                equation_sub_menu.add_command(
                    label="Insert",
                    command=equation_p
                )
                equation_sub_menu.add_command(
                    label="Delete",
                    command=equation_delete_p
                )
        except FileNotFoundError:
            pass
        except AttributeError:
            pass

    def insert_equation(self, equation):
        """Inserts value of equation in currently visible scrolled text box

        Calls check_current_editor() method from controller and assigns return to current editor. This returns the
        scrolled text box widget in the currently selected tab in the currently visible tex editor frame. Inserts value
        of equation, determined by which saved equation is chosen, at current cursor position.

        Parameters
        ----------
        equation : str
            LaTeX code to insert
        """
        current_editor = self.controller.check_current_editor()
        current_editor.insert(tk.INSERT, equation)

    def delete_equation(self, equation):
        """Deletes equation from saved_equations.py and saved_equations_button_menu

        Removes the entry from the equations dictionary inside the file files/saved_equations.py with key equal to the
        value of equation argument. Opens the saved_equations.py file and overwrites its contents with the updated
        dictionary. Deletes the menu with name equal to the value of equation from the saved equations menu

        Parameters
        ----------
        equation : str
            LaTeX code to insert
        """
        equations = saved_equations.equations
        equations.pop(equation)
        file = open("files/saved_equations.py", "w")
        file.write(f"equations = {equations}")
        file.close()
        self.saved_equations_button_menu.delete(equation)


from django.contrib.auth.tokens import default_token_generator as token_generator
from django.contrib.sites.shortcuts import get_current_site
from django.core.mail import send_mail
from django.template.loader import render_to_string
from django.utils.encoding import force_bytes
from django.utils.http import urlsafe_base64_encode

from config.settings import EMAIL_HOST_USER

def send_verify_email(request, user):
    current_site = get_current_site(request)
    context = {
        'user': user,
        'domain': current_site.domain,
        'uid': urlsafe_base64_encode(force_bytes(user.pk)),
        'token': token_generator.make_token(user),
    }

    email_body = f"Подтвердите свой адрес электронной почты, перейдя по ссылке: " \
                 f"http://{context['domain']}/users/verify_email/{context['uid']}/{context['token']}/"
    send_mail(
        subject='Подтверждение адреса электронной почты',
        message=email_body,
        from_email=EMAIL_HOST_USER,
        recipient_list=[user.email],
        fail_silently=False
    )


from utils import relu
import torch

class alphatron_multi_relu():
    def __init__(self, x, d, kern, reluCount):
        self.x = x
        self.d = d
        self.m = len(x)
        self.kern = kern
        self.alpha = torch.FloatTensor(reluCount,self.m).zero_()
        self.reluCount=reluCount

    def perReluPredict(self, ind, x_input):
        cumu = 0
        for i in range(self.m):
            cumu += self.alpha[ind][i] * self.kern(x_input, self.x[i], self.d)
        return relu(cumu)
    def predict(self,x_input):
        cumu=0
        for i in range(self.reluCount):
            cumu+=self.perReluPredict(i,x_input)
        return cumu

    def train(self, T, train, evaluate, eta=1.0):
        eta = eta / self.m
        x = train["x"]
        y = train["y"].clone()
        for r in range(self.reluCount):
            for t in range(T):
                if t % 100 == 0:
                    print("step ", t)
                    # evaluate(train, self)
                alpha_update = self.alpha[r].clone()
                for i in range(self.m):
                    incre = eta * (y[i] - self.perReluPredict(r,x[i]))
                    alpha_update[i] += incre
                self.alpha[r].copy_(alpha_update)
            for i in range(self.m):
                y[i]-= self.perReluPredict(r,x[i])


#!/usr/bin/env python3

import os
import plistlib
import sys

try:
    import yaml
except ImportError:
    pass

def needs_yaml_quoting(s):
    return (
        s == "" or
        s[0] in "\"'%-:?@`&*!,#|>0123456789=" or
        s.startswith("<<") or
        s in ["true", "false", "null"] or
        "# " in s or
        ": " in s or
        "[" in s or
        "]" in s or
        "{" in s or
        "}" in s or
        "\n" in s or
        s[-1] in ":#" or
        s.strip() != s)

def quote(s):
    if s.count("'") <= len(s) / 2:
        return "'" + s.replace("'", "''") + "'"
    else:
        return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'

def order_keys(l):
    key_order = reversed([
        "name", "main", "match", "comment", "file_extensions",
        "first_line_match", "hidden", "match", "scope", "main"])
    l = sorted(l)
    for key in key_order:
        if key in l:
            del l[l.index(key)]
            l.insert(0, key)
    return l

def to_yaml(val, start_block_on_newline=False, indent=0):
    tab_size = 2
    out = ""

    if indent == 0:
        out += "%YAML 1.2\n---\n"
        out += "# http://www.sublimetext.com/docs/syntax.html\n"

    if isinstance(val, list):
        if len(val) == 0:
            out += "[]\n"
        else:
            if start_block_on_newline:
                out += '\n'

            for x in val:
                out += " " * indent
                out += "- "
                out += to_yaml(x, False, indent + 2)
    elif isinstance(val, dict):
        if start_block_on_newline:
            out += '\n'

        first = True
        for k in order_keys(val.keys()):
            v = val[k]

            if not first or start_block_on_newline:
                out += " " * indent
            else:
                first = False

            if isinstance(k, int):
                out += str(k)
            elif needs_yaml_quoting(k):
                out += quote(k)
            else:
                out += k

            out += ": "
            out += to_yaml(v, True, indent + tab_size)
    elif isinstance(val, str):
        if needs_yaml_quoting(val):
            if "\n" in val:
                assert(start_block_on_newline)
                if start_block_on_newline:
                    if val[-1] == "\n":
                        out += '|\n'
                    else:
                        out += "|-\n"
                for l in val.splitlines():
                    out += " " * indent
                    out += l
                    out += "\n"
            else:
                out += quote(val)
                out += "\n"
        else:
            out += val
            out += "\n"
    elif isinstance(val, bool):
        if val:
            out += "true\n"
        else:
            out += "false\n"
    else:
        out += str(val) + "\n"
    return out

def build_scope_map():
    syntax_by_scope = {}
    for f in sublime.find_resources("*.tmLanguage"):
        try:
            s = sublime.load_resource(f)
            l = plistlib.readPlistFromBytes(s.encode("utf-8"))
            if "scopeName" in l:
                fname = os.path.splitext(f)[0] + ".sublime-syntax"
                syntax_by_scope[l["scopeName"]] = os.path.basename(fname)
        except:
            pass

    return syntax_by_scope

scope_map = {}

def syntax_for_scope(key):
    use_scope_refs = True
    if use_scope_refs:
        return "scope:" + key
    else:
        global scope_map
        if len(scope_map) == 0:
            scope_map = build_scope_map()

        return scope_map[key]

def is_external_syntax(key):
    return key[0] not in "#$"

def format_external_syntax(key):
    assert(is_external_syntax(key))

    if '#' in key:
        syntax, rule = key.split('#')
        return syntax_for_scope(syntax) + '#' + rule
    else:
        return syntax_for_scope(key)

def leading_whitespace(s):
    return s[0:len(s) - len(s.lstrip())]

def format_regex(s):
    if "\n" in s:
        lines = s.splitlines()

        # trim common indentation off of each line
        if len(lines) > 1:
            common_indent = leading_whitespace(lines[1])
            for l in lines[2:]:
                cur_indent = leading_whitespace(l)
                if cur_indent.startswith(common_indent):
                    pass
                elif common_indent.startswith(cur_indent):
                    common_indent = cur_indent
                else:
                    common_indent = ""

            # Generally the first line doesn't have any indentation, add some
            if not lines[0].startswith(common_indent):
                lines[0] = common_indent + lines[0].lstrip()
        else:
            common_indent = leading_whitespace(lines[0])

        s = "\n".join([l[len(common_indent):] for l in lines]).rstrip("\n")

    return s

def format_comment(s):
    s = s.strip().replace("\t", "    ")

    if "\n" in s:
        s = s.rstrip("\n") + "\n"

    return s

def format_captures(c):
    ret = {}
    for k, v in c.items():
        if "name" not in v:
            warn("patterns and includes are not supported within captures: " + str(c))
            continue

        try:
            ret[int(k)] = v["name"]
        except ValueError:
            warn("named capture used, this is unsupported")
            ret[k] = v["name"]
    return ret

def warn(msg):
    print(msg, file=sys.stderr)

def make_context(patterns, repository):
    ctx = []
    for p in patterns:
        if "begin" in p:
            entry = {}
            entry["match"] = format_regex(p["begin"])
            if "beginCaptures" in p or "captures" in p:
                if "beginCaptures" in p:
                    captures = format_captures(p["beginCaptures"])
                else:
                    captures = format_captures(p["captures"])

                if "0" in captures:
                    entry["scope"] = captures["0"]
                    del captures["0"]

                if len(captures) > 0:
                    entry["captures"] = captures

            end_entry = {}
            end_entry["match"] = format_regex(p["end"])
            end_entry["pop"] = True
            if "endCaptures" in p or "captures" in p:
                captures = format_captures(
                    p.get("endCaptures", p.get("captures")))
                if "0" in captures:
                    end_entry["scope"] = captures["0"]
                    del captures["0"]

                if len(captures) > 0:
                    end_entry["captures"] = captures

            if "\\G" in end_entry["match"]:
                warn(
                    "pop pattern contains \\G, this will not work as expected"
                    " if it's intended to refer to the begin regex: " +
                    end_entry["match"])

            apply_last = False
            if "applyEndPatternLast" in p and p["applyEndPatternLast"] == 1:
                apply_last = True

            if "patterns" in p:
                child_patterns = p["patterns"]
            else:
                child_patterns = []

            child = make_context(child_patterns, repository)
            if apply_last:
                child.append(end_entry)
            else:
                child.insert(0, end_entry)

            if "contentName" in p:
                child.insert(0, {"meta_content_scope": p["contentName"]})
            if "name" in p:
                child.insert(0, {"meta_scope": p["name"]})

            if "comment" in p:
                comment = format_comment(p["comment"])
                if len(comment) > 0:
                    entry["comment"] = comment

            entry["push"] = child

            ctx.append(entry)

        elif "match" in p:
            entry = {}
            entry["match"] = format_regex(p["match"])

            if "name" in p:
                entry["scope"] = p["name"]

            if "captures" in p:
                entry["captures"] = format_captures(p["captures"])

            if "comment" in p:
                comment = format_comment(p["comment"])
                if len(comment) > 0:
                    entry["comment"] = comment

            ctx.append(entry)

        elif "include" in p:
            key = p["include"]

            if key[0] == '#':
                key = key[1:]
                if key not in repository:
                    raise Exception("no entry in repository for " + key)

                ctx.append({"include": key})
            elif key == "$self":
                ctx.append({"include": "main"})
            elif key == "$base":
                ctx.append({"include": "$top_level_main"})
            elif key[0] == '$':
                raise Exception("unknown include: " + key)
            else:
                # looks like an external include
                # assert(is_external_syntax(key))
                # ctx.append({"include-syntax": key})
                ctx.append({"include": format_external_syntax(key)})

        else:
            raise Exception("unknown pattern type: %s" % p.keys())

    return ctx

def convert(fname):
    if fname.startswith('Packages/'):
        s = sublime.load_resource(fname)
    else:
        with open(fname, 'r', encoding='utf-8') as f:
            s = f.read()
    l = plistlib.readPlistFromBytes(s.encode("utf-8"))

    if "repository" in l:
        repository = l["repository"]
    else:
        repository = {}

    # normalize the repository values into being a list of patterns
    for key, value in repository.items():
        if "begin" in value or "match" in value:
            repository[key] = [value]
        else:
            repository[key] = value["patterns"]

    contexts = {"main": make_context(l["patterns"], repository)}

    for key, value in repository.items():
        assert(key != "main")

        contexts[key] = make_context(value, repository)

    syn = {}

    if "comment" in l:
        comment = format_comment(l["comment"])
        if len(comment) > 0:
            syn["comment"] = comment

    if "name" in l:
        syn["name"] = l["name"]

    if "scopeName" in l:
        syn["scope"] = l["scopeName"]

    if "fileTypes" in l:
        syn["file_extensions"] = l["fileTypes"]

    if "firstLineMatch" in l:
        syn["first_line_match"] = l["firstLineMatch"]

    if "hideFromUser" in l:
        syn["hidden"] = l["hideFromUser"]

    if "hidden" in l:
        syn["hidden"] = l["hidden"]
    elif fname.endswith('.hidden-tmLanguage'):
        syn["hidden"] = True

    syn["contexts"] = contexts

    return syn

def extract_by_key(key, v):
    if isinstance(v, list):
        ret = []
        for x in v:
            ret.extend(extract_by_key(key, x))
        return ret
    elif isinstance(v, dict):
        ret = []
        for k, x in v.items():
            if k == key:
                ret.append(x)
            else:
                ret.extend(extract_by_key(key, x))
        return ret
    else:
        return []

def find_external_refs():
    refmap = {}
    for f in sublime.find_resources("*.tmLanguage"):
        l = convert(f)
        refmap[f] = extract_by_key("syntax", l)
    print(to_yaml(refmap, False, 0))

try:
    import sublime
    import sublime_plugin

    class ConvertSyntaxCommand(sublime_plugin.WindowCommand):
        def run(self):
            view = self.window.active_view()
            if not view:
                return

            syn = self.syntax_info(view)
            if not syn:
                return
            base, ext = os.path.splitext(syn)

            data = to_yaml(convert(syn))

            # to_yaml will leave some trailing whitespace, remove it
            data = "\n".join(l.rstrip() for l in data.splitlines()) + "\n"

            v = self.window.new_file()
            v.set_name(os.path.basename(base) + ".sublime-syntax")
            v.assign_syntax("Packages/YAML/YAML.sublime-syntax")
            v.settings().set("default_dir", os.path.join(sublime.packages_path(), "User"))
            v.run_command('append', {'characters': data})

        def is_visible(self):
            view = self.window.active_view()
            if not view:
                return False

            syn = self.syntax_info(view)
            return syn is not None

        def description(self):
            view = self.window.active_view()
            if not view:
                return super().description()

            syn = self.syntax_info(view)
            if not syn:
                return super().description()

            return "New Syntax from " + os.path.basename(syn) + "…"

        def syntax_info(self, view):
            extensions = ['.tmLanguage', '.hidden-tmLanguage']

            fname = view.file_name()
            if fname and os.path.splitext(fname)[1] in extensions:
                return fname

            syn = view.settings().get('syntax')
            if os.path.splitext(syn)[1] in extensions:
                return syn

            return None

except ImportError:
    import fnmatch

    class sublime:
        base_path = "."

        @classmethod
        def find_resources(cls, pattern):
            paths = []
            for root, dirs, files in os.walk(cls.base_path):
                for fname in files:
                    if fnmatch.fnmatch(fname, pattern):
                        path = os.path.join(root, fname)

                        if path[0:2] == "./" or path[0:2] == ".\\":
                            path = path[2:]

                        paths.append(path)
            return paths

        @staticmethod
        def load_resource(fname):
            with open(fname, "r", encoding="utf-8") as f:
                return f.read()

if __name__ == "__main__":
    args = sys.argv[1:]

    if not args:
        print("usage: convert_syntax.py files")
        print("       convert_syntax.py folder")
    else:
        filenames = []
        for path in args:
            if os.path.isdir(path):
                sublime.base_path = path
                filenames.extend(sublime.find_resources("*.tmLanguage"))
            else:
                filenames.append(path)

        for fname in filenames:
            outfile = os.path.splitext(fname)[0] + ".sublime-syntax"
            if os.path.exists(outfile):
                print("file already exists: " + outfile)
                continue

            data = convert(fname)
            text = to_yaml(data)
            # verify that to_yaml produces valid yaml for this object
            if 'yaml' in sys.modules:
                assert(data == yaml.load(text))

            with open(outfile, "w", encoding="utf-8") as f:
                # to_yaml will leave some trailing whitespace, remove it
                text = "\n".join(l.rstrip() for l in text.splitlines()) + "\n"
                f.write(text)

            print("converted " + outfile)


import logging
from pathlib import Path

import pytest

logging.getLogger("snowflake.connector").setLevel(logging.ERROR)

def pytest_addoption(parser):
    parser.addoption("--disable_sql_simplifier", action="store_true", default=False)

def pytest_collection_modifyitems(items) -> None:
    """Applies tags to tests based on folders that they are in."""
    top_test_dir = Path(__file__).parent
    top_doctest_dir = top_test_dir.parent.joinpath("src/snowflake/snowpark")
    for item in items:
        item_path = Path(str(item.fspath)).parent
        try:
            relative_path = item_path.relative_to(top_test_dir)
            for part in relative_path.parts:
                item.add_marker(part)
        except ValueError as e:
            # item_path.relative_to(top_test_dir) will throw an error if
            # the path isn't in the tests dir. We also accept doctest files
            # in src/snowflake/snowpark (and set a marker for them) but
            # we raise an exception for all other dirs that are passed in
            if item_path == top_doctest_dir:
                item.add_marker("doctest")
            else:
                raise e

@pytest.fixture(scope="session")
def sql_simplifier_enabled(pytestconfig):
    disable_sql_simplifier = pytestconfig.getoption("disable_sql_simplifier")
    return not disable_sql_simplifier


import argparse
import os

import h5py
import numpy as np
import pandas as pd
import tables

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Script to remove subjects from HASE hdf5 files')
    parser.add_argument("-g", required=True, type=str, help="path/paths to genotype data folder")
    parser.add_argument('-study_name', type=str, required=True, default=None, help=' Study names')
    parser.add_argument('-exclude_ids', type=str, default=None,
                        help='Table with IDs to exclude from data. Should have ID column')
    args = parser.parse_args()
    print(args)

    if args.exclude_ids is not None:
        df = pd.DataFrame.from_csv(args.snp_id_inc, index_col=None)
        print(df.head())
        if 'ID' not in df.columns:
            raise ValueError('{} table does not have ID or columns'.format(args.exclude_ids))

        df['ID'] = df.ID.astype(str)

        df_ids = pd.read_hdf(os.path.join(args.g, 'individuals', args.study_name + '.h5'), 'individuals')
        df_ids['individual'] = df_ids.individual.astype(str)

        info_index = df_ids.individual.isin(df.ID)
        remove_index = np.where(info_index == True)[0]
        keep_index = np.where(info_index == False)[0]

        if len(remove_index) == 0:
            print('There is no ids to remove!')
            exit(0)
        if len(keep_index) == len(df_ids.individual):
            print("Need to remove everybody!!! ")
            exit(0)

        individuals = df_ids.individual[~remove_index]
        chunk = pd.DataFrame.from_dict({"individual": individuals})
        chunk.to_hdf(os.path.join(args.g, 'individuals', args.study_name + '.h5'), key='individuals', format='table',
                     min_itemsize=25, complib='zlib', complevel=9)

        for g_file in os.listdir(os.path.join(args.g, 'genotype')):
            print(g_file)

            data = h5py.File(os.path.join(args.g, 'genotype', g_file), 'r')['genotype'][...]
            data = data[:, keep_index]

            h5_gen_file = tables.open_file(
                os.path.join(args.g, 'genotype', g_file), 'w', title=args.study_name)

            atom = tables.Float16Atom()
            genotype = h5_gen_file.create_carray(h5_gen_file.root, 'genotype', atom,
                                                 (data.shape),
                                                 title='Genotype',
                                                 filters=tables.Filters(complevel=9, complib='zlib'))
            genotype[:] = data
            h5_gen_file.close()


#!/usr/bin/env python3

import sys
import time
import argparse
import yaml
from os import path
from datetime import datetime, timedelta
from subprocess import call, check_call
from helpers import connect_to_influxdb, create_time_clause

backup_hour = 11  # back up at 11 AM (UTC) every day

SRC_DB = 'puffer'  # source database name to restore
TMP_DB = 'puffer_tmp'  # used as a middle ground
DST_DB = None  # to be read from YAML settings

args = None

def sanity_check_influxdb(influx_client):
    db_list = influx_client.get_list_database()

    dst_db_exists = False
    for db in db_list:
        if db['name'] == DST_DB:
            dst_db_exists = True
            break

    if not dst_db_exists:
        influx_client.create_database(DST_DB)
        sys.stderr.write('destination database "{}" is created\n'
                         .format(DST_DB))

def get_files_to_restore(start_date, end_date, influx_client):
    date_format = '%Y-%m-%dT%H'
    start_date = datetime.strptime(start_date, date_format)
    end_date = datetime.strptime(end_date, date_format)

    if end_date <= start_date:
        sys.exit('END_DATE precedes START_DATE')

    ret = []

    s = start_date
    while True:
        e = s + timedelta(days=1)
        if e > end_date:
            break

        s_str = s.strftime(date_format)
        e_str = e.strftime(date_format)
        f = s_str + '_' + e_str + '.tar.gz'

        # check if the file to restore exists on cloud
        cmd = 'gsutil -q stat gs://puffer-influxdb-analytics/{}'.format(f)
        if call(cmd, shell=True) != 0:
            sys.exit('Error: {} does not exist on cloud'.format(f))

        if args.force:
            sys.stderr.write('Will restore {}\n'.format(f))
            ret.append(f)
        else:
            # check if the data within the range already exists in InfluxDB
            time_clause = create_time_clause(s, e)
            results = influx_client.query(
                'SELECT count(video_ts) FROM video_acked WHERE ' + time_clause)

            count = None
            for pt in results['video_acked']:
                if count is not None:
                    sys.exit('Error: query should only return only record')

                count = pt['count']
                sys.stderr.write('Info: {} records found within {} - {}\n'
                                 .format(count, s_str, e_str))

            if count is None:
                sys.stderr.write('Will restore {}\n'.format(f))
                ret.append(f)
            else:
                if args.allow_skipping:
                    sys.stderr.write('Warning: skipping {}\n'.format(f))
                else:
                    sys.exit('Error: does not allow skipping {}'.format(f))

        s = e

    return ret

def download_untar(f):
    d = f[:f.index('.')]
    if path.isdir(d):
        sys.stderr.write('Found {} in the current directory\n'.format(d))
        return d

    # download
    cmd = 'gsutil cp gs://puffer-influxdb-analytics/{} .'.format(f)
    check_call(cmd, shell=True)

    # untar
    cmd = 'tar xf {}'.format(f)
    check_call(cmd, shell=True)

    # return uncompressed folder name
    return d

def restore(f, influx_client):
    sys.stderr.write('Restoring {}...\n'.format(f))

    # download data from Google cloud
    d = download_untar(f)

    # restore to a temporary database (with retries)
    for retry in range(5):
        influx_client.drop_database(TMP_DB)

        cmd = ('influxd restore -portable -db {} -newdb {} {}'
               .format(SRC_DB, TMP_DB, d))
        if call(cmd, shell=True) != 0:
            continue

        influx_client.switch_database(TMP_DB)

        # workaround: sleep for a while to avoid influxdb errors
        # possible errors: shard is disabled, engine is closed
        time.sleep(5 * (retry + 1))

        try:
            # sideload the data into the destination database (with retries)
            influx_client.query(
                'SELECT * INTO {}..:MEASUREMENT FROM /.*/ GROUP BY *'
                .format(DST_DB))
            sys.stderr.write('Successfully restored data in {}\n'.format(f))
            return
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as e:
            print('Error:', e, file=sys.stderr)
            sys.stderr.write('Retrying...\n')

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('yaml_settings')
    parser.add_argument('--from', required=True, dest='start_date',
                        help='e.g., "2019-04-03" ({} AM in UTC)'.format(backup_hour))
    parser.add_argument('--to', required=True, dest='end_date',
                        help='e.g., "2019-04-05" ({} AM in UTC)'.format(backup_hour))
    parser.add_argument('--allow-skipping', action='store_true',
                        help='allow skipping a day if data already exist in InfluxDB')
    parser.add_argument('--force', action='store_true',
                        help='force restoring data even if data exist')
    parser.add_argument('--dry-run', action='store_true',
                        help='only check and print the status of InfluxDB')
    global args
    args = parser.parse_args()

    with open(args.yaml_settings, 'r') as fh:
        yaml_settings = yaml.safe_load(fh)

    global DST_DB
    DST_DB = yaml_settings['influxdb_connection']['dbname']

    # connect to InfluxDB
    influx_client = connect_to_influxdb(yaml_settings)
    sanity_check_influxdb(influx_client)

    # parse input dates and get valid files to restore
    start_date = args.start_date + 'T{}'.format(backup_hour)
    end_date = args.end_date + 'T{}'.format(backup_hour)
    files_to_restore = get_files_to_restore(start_date, end_date, influx_client)

    if not args.dry_run:
        for f in files_to_restore:
            restore(f, influx_client)

if __name__ == '__main__':
    main()


import copy
import logging
import json

# External packages
import jsonfield
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db import models, IntegrityError

# SCION
from lib.types import ServiceType
from lib.defines import DEFAULT_MTU
from lib.packet.scion_addr import ISD_AS

# SCION-WEB
from ad_manager.util.common import empty_dict
from ad_manager.util.defines import (
    DEFAULT_BANDWIDTH,
    SCION_SUGGESTED_PORT,
)

PORT = SCION_SUGGESTED_PORT
PACKAGE_DIR_PATH = 'gen'

class SelectRelatedModelManager(models.Manager):
    """
    Model manager that also selects related objects from the database,
    avoiding multiple similar queries.
    """

    def get_queryset(self):
        queryset = super(SelectRelatedModelManager, self).get_queryset()
        related_fields = getattr(self.model, 'related_fields', [])
        if not related_fields:
            return queryset.select_related()
        else:
            return queryset.select_related(*related_fields)

class OrganisationAdmin(models.Model):
    user = models.OneToOneField(User)
    is_org_admin = models.BooleanField(default=False)
    account_id = models.CharField(max_length=260, null=False, blank=True)
    secret = models.CharField(max_length=260, null=False, blank=True)

class ISD(models.Model):
    id = models.IntegerField(primary_key=True)

    def get_absolute_url(self):
        return reverse('isd_detail', args=[self.id])

    def __str__(self):
        return str(self.id)

    class Meta:
        verbose_name = 'ISD'
        ordering = ['id']

def transform_integer_ids_in_topology(topo):
    """
    Transform the topology so that keys that are string representations
    of integers are replaced with their integers. E.g. "1" replaced with 1
    These transformations are needed by the generating code of scionproto
    because it expects some keys to be integers and not strings
    """
    assert isinstance(topo, dict), 'Topology should be a dictionary'
    jsontopo = json.dumps(topo)
    def jsonhook(d):
        ret = {}
        for k,v in d.items():
            try:
                k = int(k)
            except:
                pass
            ret[k] = v
        return ret
    topo = json.loads(jsontopo, object_hook=jsonhook)
    return topo

class AD(models.Model):
    as_id = models.IntegerField(default=-1)
    isd = models.ForeignKey('ISD')
    as_id_str = models.CharField(max_length=15, null=True)
    is_core_ad = models.BooleanField(default=False)
    simple_conf_mode = models.BooleanField(default=False)
    is_open = models.BooleanField(default=True)
    commit_hash = models.CharField(max_length=100, default='')
    md_host = models.GenericIPAddressField(default='127.0.0.1')
    original_topology = jsonfield.JSONField(default=empty_dict)
    sig_pub_key = models.CharField(max_length=100, null=True, blank=True)
    sig_priv_key = models.CharField(max_length=100, null=True, blank=True)
    enc_pub_key = models.CharField(max_length=100, null=True, blank=True)
    enc_priv_key = models.CharField(max_length=100, null=True, blank=True)
    master_as_key = models.CharField(max_length=100, null=True, blank=True)
    certificate = models.TextField(null=True, blank=True)
    trc = models.TextField(null=True, blank=True)
    keys = jsonfield.JSONField(default=empty_dict)
    core_keys = jsonfield.JSONField(default=empty_dict)

    # Use custom model manager with select_related()
    objects = SelectRelatedModelManager()

    @classmethod
    def from_db(cls, db, field_names, values):
        """Create an AD instance from the DB. Just transform the topology"""
        ad = super().from_db(db, field_names, values)
        ad.original_topology = transform_integer_ids_in_topology(ad.original_topology)
        return ad

    class Meta:
        unique_together = (("as_id", "isd"),)
        verbose_name = 'AD'
        ordering = ['as_id_str']

    def generate_topology_dict(self):
        """
        Create a Python dictionary with the stored AS topology.
        """
        assert isinstance(self.original_topology, dict)
        out_dict = copy.deepcopy(self.original_topology)
        out_dict.update({
            'ISDID': int(self.isd_id), 'ADID': int(self.as_id),
            'Core': int(self.is_core_ad),
            'BorderRouters': {}, 'PathService': {}, 'BeaconService': {},
            'CertificateService': {},
        })
        for router in self.borderrouter_set.all():
            out_dict['BorderRouters'][router.name] = router.get_dict()
        for service in self.service_set.all():
            if service.name.startswith('bs'):
                out_dict['BeaconService'][service.name] = service.get_dict()
            elif service.name.startswith('ps'):
                out_dict['PathService'][service.name] = service.get_dict()
            elif service.name.startswith('cs'):
                out_dict['CertificateService'][service.name] = service.get_dict()
        return out_dict

    def get_all_elements(self):
        element_groups = [self.borderrouter_set.all(), self.service_set.all()]
        for element_group in element_groups:
            for element in element_group:
                yield element

    def get_all_element_ids(self):
        all_elements = self.get_all_elements()
        element_ids = [element.id_str() for element in all_elements]
        return element_ids

    def fill_router_info(self, router_dict):
        """
        Update the router information in the database.
        (i.e., BorderRouter and BorderRouterAddress.)
        :param dict router_dict: topo_dict['BorderRouters']
        """
        for name, router in router_dict.items():
            br_obj, _ = BorderRouter.objects.update_or_create(name=name, ad=self)
            router_addr = router['InternalAddrs']['IPv4']
            addr_info = router_addr['PublicOverlay']
            internal_public_obj, _ = BorderRouterAddress.objects.update_or_create(
                addr=addr_info['Addr'],
                l4port=addr_info['OverlayPort'],
                addr_type='IPv4',
                is_public=True,
                router=br_obj,
                ad=self
            )
            addr_info = router_addr.get('BindOverlay', None)
            if addr_info:
                internal_bound_obj, _ = BorderRouterAddress.objects.update_or_create(
                addr=addr_info['Addr'],
                l4port=addr_info['OverlayPort'],
                addr_type='IPv4',
                is_public=False,
                router=br_obj,
                ad=self
            )
            else:
                internal_bound_obj = None
            # TODO: we need to update the model: BorderRouterAddress should have also CtrlAddr, both should
            # have public + bind address. BorderRouterInterface should not have an index
            # But I don't find where we read this data for anything !

            for if_id, intf in router["Interfaces"].items():
                isd_as = ISD_AS(intf["ISD_AS"])
                br_addr_obj = internal_public_obj
                br_inft_obj, _ = BorderRouterInterface.objects.update_or_create(
                    addr=intf['PublicOverlay']['Addr'],
                    l4port=intf['PublicOverlay']['OverlayPort'],
                    remote_addr=intf['RemoteOverlay']['Addr'],
                    remote_l4port=intf['RemoteOverlay']['OverlayPort'],
                    internal_addr_idx=0,
                    interface_id=if_id,
                    bandwidth=intf['Bandwidth'],
                    mtu=intf['MTU'],
                    neighbor_isd_id=isd_as[0],
                    neighbor_as_id=isd_as[1],
                    neighbor_as_id_str=isd_as.as_str(),
                    neighbor_type=intf["LinkTo"],
                    router_addr=br_addr_obj,
                    ad=self,
                    bind_addr=intf['BindOverlay']['Addr'] if 'BindOverlay' in intf.keys() else None,
                    bind_l4port=intf['BindOverlay']['OverlayPort'] if 'BindOverlay' in intf.keys() else None,
                )

    def fill_service_info(self, service_dict):
        """
        Update the service information in the database.
        (i.e., Service and ServiceAddress tables.)
        :param dict service_dict: (e.g., topo_dict['BeaconService'])
        """
        for name, service in service_dict.items():
            srv_obj, _ = Service.objects.update_or_create(name=name, ad=self)
            service_addrs = service['Addrs']['IPv4']
            addr_info = service_addrs['Public']
            srv_public_obj, _ = ServiceAddress.objects.update_or_create(
                addr=addr_info['Addr'],
                l4port=addr_info['L4Port'],
                addr_type='IPv4',
                is_public=True,
                service=srv_obj,
                ad=self
            )
            addr_info = service_addrs.get('Bind', None)
            if addr_info:
                srv_bind_obj, _ = ServiceAddress.objects.update_or_create(
                    addr=addr_info['Addr'],
                    l4port=addr_info['L4Port'],
                    addr_type='IPv4',
                    is_public=False,
                    service=srv_obj,
                    ad=self
                )

    def fill_from_topology(self, topology_dict, clear=False, auto_refs=False):
        """
        Add infrastructure elements (servers, routers) to the AD, extracted
        from the topology dictionary.
        """
        assert isinstance(topology_dict, dict), 'Dictionary expected'
        if clear:
            self.borderrouter_set.all().delete()
            self.borderrouteraddress_set.all().delete()
            self.borderrouterinterface_set.all().delete()
            self.service_set.all().delete()
            self.serviceaddress_set.all().delete()

        self.original_topology = topology_dict
        self.is_core_ad = (topology_dict['Core'] == 1)
        self.save()

        routers = topology_dict["BorderRouters"]
        beacon_servers = topology_dict["BeaconService"]
        certificate_servers = topology_dict["CertificateService"]
        path_servers = topology_dict["PathService"]

        try:
            self.fill_router_info(routers)
            self.fill_service_info(beacon_servers)
            self.fill_service_info(certificate_servers)
            self.fill_service_info(path_servers)

        except IntegrityError:
            logging.warning("Integrity error in AD.fill_from_topology(): "
                            "ignoring")
            raise

    def fill_cloud_info(self, topology_params):
        """
        Update cloud machine information in the database
        :param QueryDict topology_params
        """
        self.commit_hash = topology_params['commitHash']
        self.save()
        unique_addr = topology_params.getlist('inputCloudAddress')
        providers = topology_params.getlist('inputCloudEngine')
        host_name = topology_params.getlist('inputHostname')
        for i in range(len(unique_addr)):
            cloud_obj, _ = CloudMachine.objects.update_or_create(
                addr=unique_addr[i],
                host_name=host_name[i],
                cloud_provider=providers[i],
                ad=self
            )

    def get_absolute_url(self):
        return reverse('ad_detail', args=[self.as_id])

    def get_full_process_name(self, id_str):
        if ':' in id_str:
            return id_str
        else:
            # changed for rpc log retrieval, to match new supervisord names
            return "as{}-{}:{}".format(self.isd.id, self.as_id, id_str)

    def __str__(self):
        return '{}-{}'.format(self.isd.id, self.as_id)

class AddressElement(models.Model):
    addr = models.GenericIPAddressField()
    l4port = models.IntegerField(default=-1)
    overlay_port = models.IntegerField(null=True)
    addr_type = models.CharField(max_length=5, default="IPv4")
    is_public = models.BooleanField(default=True)
    ad = models.ForeignKey(AD)

    class Meta:
        abstract = True

class Service(models.Model):
    ad = models.ForeignKey(AD)
    name = models.CharField(max_length=20, null=True)

class ServiceAddress(AddressElement):
    service = models.ForeignKey(Service)

class BorderRouter(models.Model):
    ad = models.ForeignKey(AD)
    name = models.CharField(max_length=20, null=True)

class BorderRouterAddress(AddressElement):
    router = models.ForeignKey(BorderRouter)

class BorderRouterInterface(models.Model):
    addr = models.GenericIPAddressField()
    l4port = models.IntegerField(default=-1)
    bind_addr = models.GenericIPAddressField(default=None, null=True)
    bind_l4port = models.IntegerField(default=None, null=True)
    remote_addr = models.GenericIPAddressField(null=True)
    remote_l4port = models.IntegerField(null=True)
    internal_addr_idx = models.IntegerField()
    interface_id = models.IntegerField()
    bandwidth = models.IntegerField()
    mtu = models.IntegerField()
    NEIGHBOR_TYPES = (
        ('CHILD',) * 2,
        ('PARENT',) * 2,
        ('PEER',) * 2,
        ('CORE',) * 2,
    )
    neighbor_isd_id = models.IntegerField(null=True)
    neighbor_as_id = models.IntegerField(null=True)
    neighbor_as_id_str = models.CharField(max_length=15, null=True)
    neighbor_type = models.CharField(max_length=10, choices=NEIGHBOR_TYPES)
    router_addr = models.ForeignKey(BorderRouterAddress)
    ad = models.ForeignKey(AD)

class CloudMachine(models.Model):
    addr = models.GenericIPAddressField()
    host_name = models.CharField(max_length=20, null=True)
    cloud_provider = models.CharField(max_length=20, null=True)
    ad = models.ForeignKey(AD)

class SCIONWebElement(models.Model):
    addr = models.GenericIPAddressField()
    port = models.IntegerField()
    addr_internal = models.GenericIPAddressField(default=None, null=True)
    port_internal = models.IntegerField(default=None, null=True)
    ad = models.ForeignKey(AD)
    name = models.CharField(max_length=20, null=True)

    def id_str(self):
        # FIXME How to identify multiple servers of the same type?
        # return "{}{}-{}-{}".format(self.prefix, self.ad.isd_id,
        #                           self.ad_id, self.name)
        return self.name

    def get_dict(self):
        return {'AddrType': 'IPV4', 'Addr': self.addr}

    def __str__(self):
        return '{} -- {}'.format(self.ad, self.addr)

    class Meta:
        abstract = True

class BeaconServerWeb(SCIONWebElement):
    prefix = ServiceType.BS

    class Meta:
        verbose_name = 'Beacon server'
        unique_together = (("ad", "addr", "port"),)

class CertificateServerWeb(SCIONWebElement):
    prefix = ServiceType.CS

    class Meta:
        verbose_name = 'Certificate server'
        unique_together = (("ad", "addr", "port"),)

class PathServerWeb(SCIONWebElement):
    prefix = ServiceType.PS

    class Meta:
        verbose_name = 'Path server'
        unique_together = (("ad", "addr", "port"),)

class RouterWeb(SCIONWebElement):
    NEIGHBOR_TYPES = (
        ('CHILD',) * 2,
        ('PARENT',) * 2,
        ('PEER',) * 2,
        ('CORE',) * 2,
    )

    neighbor_isd_id = models.IntegerField(null=True)
    neighbor_as_id = models.IntegerField(null=True)
    neighbor_type = models.CharField(max_length=10, choices=NEIGHBOR_TYPES)

    interface_id = models.IntegerField()
    interface_addr = models.GenericIPAddressField()
    interface_port = models.IntegerField()
    # Allow the toaddr and toport be null since the user might not yet
    # have the complete remote router information.
    interface_toaddr = models.GenericIPAddressField(null=True)
    interface_toport = models.IntegerField(null=True)

    def get_dict(self):
        out_dict = super(RouterWeb, self).get_dict()
        out_dict['Interface'] = {'NeighborType': self.neighbor_type,
                                 'NeighborISD': self.neighbor_isd_id,
                                 'NeighborAD': self.neighbor_as_id,
                                 'Addr': str(self.interface_addr),
                                 'AddrType': 'IPV4',
                                 'ToAddr': str(self.interface_toaddr),
                                 'UdpPort': self.interface_port,
                                 'ToUdpPort': self.interface_toport,
                                 'IFID': self.interface_id,
                                 }
        return out_dict

    class Meta:
        verbose_name = 'Router'
        unique_together = (("ad", "addr", "port"),)

# Sibra service is no longer available until a new sibra service is delivered
"""
class SibraServerWeb(SCIONWebElement):
    prefix = SIBRA_SERVICE

    class Meta:
        verbose_name = 'SIBRA server'
        unique_together = (("ad", "addr", "port"),)
"""

class JoinRequest(models.Model):
    STATUS_OPTIONS = ['NONE', 'SENT', 'ACCEPTED', 'DECLINED']
    created_by = models.ForeignKey(User)

    isd_to_join = models.IntegerField(default=-1)
    join_as_a_core = models.BooleanField(default=False)
    status = models.CharField(max_length=20,
                              choices=zip(STATUS_OPTIONS, STATUS_OPTIONS),
                              default='NONE')

    sig_pub_key = models.CharField(max_length=100, null=True, blank=True)
    sig_priv_key = models.CharField(max_length=100, null=True, blank=True)
    enc_pub_key = models.CharField(max_length=100, null=True, blank=True)
    enc_priv_key = models.CharField(max_length=100, null=True, blank=True)

    certificate = models.CharField(max_length=1000, null=True, blank=True)
    trc = models.CharField(max_length=500, null=True, blank=True)

    def is_accepted(self):
        return self.status == 'ACCEPTED'

class ConnectionRequest(models.Model):
    STATUS_OPTIONS = ['NONE', 'SENT', 'APPROVED', 'DECLINED']
    LINK_TYPE = ['PARENT', 'CHILD', 'PEER', 'CORE']
    OVERLAY_TYPE = ['IPv4', 'IPv6', 'UDP/IPv4', 'UDP/IPv6']

    created_by = models.ForeignKey(User)
    connect_to = models.CharField(max_length=100, null=True, blank=True)
    connect_from = models.ForeignKey(AD, blank=True, null=True)
    info = models.TextField()
    router_public_ip = models.GenericIPAddressField()
    router_public_port = models.IntegerField(default=int(PORT))
    # router_info is the IP (and port) shown in the ConnectionRequest form
    # to select from.
    router_info = models.TextField(null=True)
    mtu = models.IntegerField(null=True, default=DEFAULT_MTU)
    bandwidth = models.IntegerField(null=True, default=DEFAULT_BANDWIDTH)
    link_type = models.CharField(max_length=20,
                                 choices=zip(LINK_TYPE, LINK_TYPE),
                                 default='PARENT')
    overlay_type = models.CharField(max_length=20,
                                    choices=zip(OVERLAY_TYPE, OVERLAY_TYPE),
                                    default='UDP/IPv4')
    status = models.CharField(max_length=20,
                              choices=zip(STATUS_OPTIONS, STATUS_OPTIONS),
                              default='NONE')

    related_fields = ('connect_from__isd', 'created_by')
    objects = SelectRelatedModelManager()

    def is_approved(self):
        return self.status == 'APPROVED'


"""
Module for the Window class which handles everything related to window drawing and updating.
Game wide objects and settings should be set here.
"""

import pygame

class Window:
    """
    Main class for handling everything window related.
    """
    def __init__(self, game: "Game"):
        """
        :param game: Game main object for current game
        """
        self.game = game
        self.screen = self.game.screen

    def update(self) -> None:
        """
        Method loads the current page and everything else that should be drawn to the screen.
        """
        self.screen.fill((0, 0, 0))
        self.game.controller.current_page.update()
        self.game.controller.current_page.draw()
        self.game.development.draw()  # Draw development
        pygame.display.update()


import behave

@behave.when(u'I delete the dataset that was created by name')
def step_impl(context):
    context.project.datasets.delete(dataset_name=context.dataset.name,
                                    sure=True,
                                    really=True)

@behave.when(u'I delete the dataset that was created by id')
def step_impl(context):
    context.project.datasets.delete(dataset_id=context.dataset.id,
                                    sure=True,
                                    really=True)

@behave.when(u'I try to delete a dataset by the name of "{dataset_name}"')
def step_impl(context, dataset_name):
    try:
        context.project.datasets.delete(dataset_name=dataset_name,
                                        sure=True,
                                        really=True)
        context.error = None
    except Exception as e:
        context.error = e

@behave.then(u'No dataset was deleted')
def step_impl(context):
    assert len(context.project.datasets.list()) == context.dataset_count


from feature.db_interface import DBInterface
import math
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import math

SIMILAR = 0
DIFFERENT = 90

class TopicFeatures:
    def get_cos_similarity_for_article(self, comment, article):
        cos_sim_in_degree = SIMILAR
        corpus = [comment.strip()]
        if article is str and len(article) > 0 and type(comment) is str:
            vector = TfidfVectorizer(min_df=1)
            corpus.extend([article])
            print(corpus)
            vector.fit(corpus)

            tfidf_comment = vector.transform([comment])
            tfidf_article = vector.transform([article])

            cos_sim = cosine_similarity(tfidf_comment, tfidf_article)
            cos_sim_in_degree = self._cos_to_degree(cos_sim)

        return cos_sim_in_degree

    def get_cos_similarity_for_hate_comments_of_article(self, comment, hate_comments):
        cos_sim_in_degree = DIFFERENT
        corpus = [comment.strip()]

        if len(hate_comments) != 0 and type(comment) is str:

            comment = comment + ' nostop'
            hate_comments_corpus = 'nostop'
            for hate_comment in hate_comments:
                hate_comments_corpus += ' ' + hate_comment

            corpus.extend([hate_comments_corpus])

            vector = TfidfVectorizer(min_df=1)
            vector.fit(corpus)

            tfidf_comment = vector.transform([comment])
            tfidf_hate_comments = vector.transform([hate_comments_corpus])

            cos_sim = cosine_similarity(tfidf_comment, tfidf_hate_comments)
            cos_sim_in_degree = self._cos_to_degree(cos_sim)

        return cos_sim_in_degree

    def _cos_to_degree(self, cos):
        if cos <= 1.0 and cos >= 0:
            return math.degrees(math.acos(cos))

        return 0


"""
This module contains the definitions of classes hls4ml uses to represent data types. The data types are equivalents of
C++/HLS data types. The basic type(``PrecisionType``) is defined as having a specified width in bits (it's 'precision').
The Precision types are given names for convenience (``NamedType``). Named types are the building blocks of
higher-dimensional tensors, which are defined as arrays or FIFO streams in the generated code.
"""

from enum import Enum

import numpy as np
import tensorflow as tf
from qkeras.quantizers import get_quantizer

# region Quantizer definition

class Quantizer:
    """
    Base class for representing quantizers in hls4ml.

    Subclasses of ``Quantizer`` are expected to wrap the quantizers of upstream tools (e.g., QKeras).

    Args:
        bits (int): Total number of bits used by the quantizer.
        hls_type (NamedType): The hls4ml type used by the quantizer.
    """

    def __init__(self, bits, hls_type):
        self.bits = bits
        self.hls_type = hls_type

    def __call__(self, data):
        raise NotImplementedError

class BinaryQuantizer(Quantizer):
    """Quantizer that quantizes to 0 and 1 (``bits=1``) or -1 and 1 (``bits==2``).

    Args:
        bits (int, optional): Number of bits used by the quantizer. Defaults to 2.

    Raises:
        Exception: Raised if ``bits>2``
    """

    def __init__(self, bits=2):
        if bits == 1:
            hls_type = XnorPrecisionType()
        elif bits == 2:
            hls_type = IntegerPrecisionType(width=2)
        else:
            raise Exception(f'BinaryQuantizer suppots 1 or 2 bits, but called with bits={bits}')
        super().__init__(bits, hls_type)

    def __call__(self, data):
        zeros = np.zeros_like(data)
        ones = np.ones_like(data)
        quant_data = data
        if self.bits == 1:
            quant_data = np.where(data > 0, ones, zeros).astype('int')
        if self.bits == 2:
            quant_data = np.where(data > 0, ones, -ones)
        return quant_data

class TernaryQuantizer(Quantizer):
    """Quantizer that quantizes to -1, 0 and 1."""

    def __init__(self):
        super().__init__(2, IntegerPrecisionType(width=2))

    def __call__(self, data):
        zeros = np.zeros_like(data)
        ones = np.ones_like(data)
        return np.where(data > 0.5, ones, np.where(data <= -0.5, -ones, zeros))

class QKerasQuantizer(Quantizer):
    """Wrapper around QKeras quantizers.

    Args:
        config (dict): Config of the QKeras quantizer to wrap.
    """

    def __init__(self, config):
        self.quantizer_fn = get_quantizer(config)
        self.alpha = config['config'].get('alpha', None)
        if config['class_name'] == 'quantized_bits':
            self.bits = config['config']['bits']
            self.hls_type = self._get_type(config)
        # ! includes stochastic_ternary
        elif 'ternary' in config['class_name']:
            self.bits = 2
            self.hls_type = IntegerPrecisionType(width=2, signed=True)
        # ! includes stochastic_binary
        elif 'binary' in config['class_name']:
            self.bits = 1
            self.hls_type = XnorPrecisionType()
        else:
            print("Unsupported quantizer: " + config['class_name'])
            self.bits = 16
            self.hls_type = FixedPrecisionType(width=16, integer=6, signed=True)

    def __call__(self, data):
        tf_data = tf.convert_to_tensor(data)
        return self.quantizer_fn(tf_data).numpy()
        # return self.quantizer_fn(data)

    def _get_type(self, quantizer_config):
        width = quantizer_config['config']['bits']
        integer = quantizer_config['config'].get('integer', 0)
        if quantizer_config['class_name'] == 'quantized_po2':
            return ExponentPrecisionType(width=width, signed=True)
        if width == integer:
            if width == 1:
                return XnorPrecisionType()
            else:
                return IntegerPrecisionType(width=width, signed=True)
        else:
            return FixedPrecisionType(width=width, integer=integer + 1, signed=True)

class QKerasBinaryQuantizer(Quantizer):
    """Wrapper around QKeras binary quantizer.

    Args:
        config (dict): Config of the QKeras quantizer to wrap.
    """

    def __init__(self, config, xnor=False):
        self.bits = 1 if xnor else 2
        self.hls_type = XnorPrecisionType() if xnor else IntegerPrecisionType(width=2, signed=True)
        self.alpha = config['config']['alpha']
        # Use the QKeras quantizer to handle any stochastic / alpha stuff
        self.quantizer_fn = get_quantizer(config)
        # Then we use our BinaryQuantizer to convert to '0,1' format
        self.binary_quantizer = BinaryQuantizer(1) if xnor else BinaryQuantizer(2)

    def __call__(self, data):
        x = tf.convert_to_tensor(data)
        y = self.quantizer_fn(x).numpy()
        return self.binary_quantizer(y)

class QKerasPO2Quantizer(Quantizer):
    """Wrapper around QKeras power-of-2 quantizers.

    Args:
        config (dict): Config of the QKeras quantizer to wrap.
    """

    def __init__(self, config):
        self.bits = config['config']['bits']
        self.quantizer_fn = get_quantizer(config)
        self.hls_type = ExponentPrecisionType(width=self.bits, signed=True)

    def __call__(self, data):
        # Weights are quantized to nearest power of two
        x = tf.convert_to_tensor(data)
        y = self.quantizer_fn(x)
        if hasattr(y, 'numpy'):
            y = y.numpy()
        return y

# endregion

# region Precision types

class RoundingMode(Enum):
    TRN = 1
    TRN_ZERO = 2
    RND = 3
    RND_ZERO = 4
    RND_INF = 5
    RND_MIN_INF = 6
    RND_CONV = 7

    def __str__(self):
        return self.name

    @classmethod
    def from_string(cls, mode):
        mode = mode.strip().replace('AP_', '').upper()
        mode = mode.strip().replace('AC_', '').upper()

        return cls[mode]

class SaturationMode(Enum):
    WRAP = 1
    SAT = 2
    SAT_ZERO = 3
    SAT_SYM = 4

    def __str__(self):
        return self.name

    @classmethod
    def from_string(cls, mode):
        mode = mode.strip().replace('AP_', '').upper()
        mode = mode.strip().replace('AC_', '').upper()

        return cls[mode]

class PrecisionType:
    """
    Base class representing a precision type of specified width.

    Subclasses of this provide concrete implementations of arbitrary precision integer and fixed-point types.

    Args:
        width (int): Number of bits used by the precision type.
        signed (bool): Signed or unsigned type.
    """

    def __init__(self, width, signed):
        self.width = width
        self.signed = signed

    def __eq__(self, other):
        eq = self.width == other.width
        eq = eq and self.signed == other.signed

class IntegerPrecisionType(PrecisionType):
    """Arbitrary precision integer  data type.

    This type is equivalent to ap_(u)int and ac_int HLS types.

    Args:
        width (int, optional): Number of bits used. Defaults to 16.
        signed (bool, optional): Signed or unsigned type. Defaults to ``True``.
    """

    def __init__(self, width=16, signed=True):
        super().__init__(width=width, signed=signed)
        self.integer = width
        self.fractional = 0

    def __str__(self):
        typestring = '{signed}int<{width}>'.format(signed='u' if not self.signed else '', width=self.width)
        return typestring

    def __eq__(self, other):
        eq = self.width == other.width
        eq = eq and self.signed == other.signed
        # These are probably unnecessary
        eq = eq and self.integer == other.integer
        eq = eq and self.fractional == other.fractional
        return eq

class FixedPrecisionType(PrecisionType):
    """Arbitrary precision fixed-point data type.

    This type is equivalent to ap_(u)fixed and ac_fixed HLS types.

    Args:
        width (int, optional): Total number of bits used. Defaults to 16.
        integer (int, optional): Number of integer bits left of the decimal point. Defaults to 6.
        signed (bool, optional): Signed or unsigned type. Defaults to ``True``.
        rounding_mode (RoundingMode, optional): Quantization mode. Defaults to ``None`` (TRN).
        saturation_mode (SaturationMode, optional): Overflow mode. Defaults to ``None`` (WRAP).
        saturation_bits (int, optional): The number of saturation bits. Defaults to ``None``.
    """

    def __init__(self, width=16, integer=6, signed=True, rounding_mode=None, saturation_mode=None, saturation_bits=None):
        super().__init__(width=width, signed=signed)
        self.integer = integer
        self.fractional = width - integer
        self.rounding_mode = rounding_mode
        self.saturation_mode = saturation_mode
        self.saturation_bits = saturation_bits

    @property
    def rounding_mode(self):
        return self._rounding_mode

    @rounding_mode.setter
    def rounding_mode(self, mode):
        if isinstance(mode, str):
            self._rounding_mode = RoundingMode.from_string(mode)
        else:
            self._rounding_mode = mode

    @property
    def saturation_mode(self):
        return self._saturation_mode

    @saturation_mode.setter
    def saturation_mode(self, mode):
        if isinstance(mode, str):
            self._saturation_mode = SaturationMode.from_string(mode)
        else:
            self._saturation_mode = mode

    def __str__(self):
        args = [self.width, self.integer, self.rounding_mode, self.saturation_mode, self.saturation_bits]
        args = ','.join([str(arg) for arg in args if arg is not None])
        typestring = '{signed}fixed<{args}>'.format(signed='u' if not self.signed else '', args=args)
        return typestring

    def __eq__(self, other):
        eq = self.width == other.width
        eq = eq and self.integer == other.integer
        eq = eq and self.fractional == other.fractional
        eq = eq and self.signed == other.signed
        eq = eq and self.rounding_mode == other.rounding_mode
        eq = eq and self.saturation_mode == other.saturation_mode
        eq = eq and self.saturation_bits == other.saturation_bits
        return eq

class XnorPrecisionType(PrecisionType):
    """
    Convenience class to differentiate 'regular' integers from BNN Xnor ones
    """

    def __init__(self):
        super().__init__(width=1, signed=False)
        self.integer = 1

    def __str__(self):
        typestring = 'uint<1>'
        return typestring

class ExponentPrecisionType(PrecisionType):
    """
    Convenience class to differentiate 'regular' integers from those which represent exponents,
    for QKeras po2 quantizers, for example.
    """

    def __init__(self, width=16, signed=True):
        super().__init__(width=width, signed=signed)

    def __str__(self):
        typestring = '{signed}int<{width}>'.format(signed='u' if not self.signed else '', width=self.width)
        return typestring

def find_minimum_width(data, signed=True):
    """
    Helper function to find the minimum integer width to express all entries in the data array
    without saturation / overflow.

    Args:
        data (ndarray): Data array.
        signed (bool, optional): Signed or unsigned type. Defaults to ``True.``

    Returns:
        int: Minimum integer width required.
    """
    maxdata = np.amax(np.abs(data))
    if maxdata == 0.0:
        # fringe case (amax(abs(data)) == 0 -> data is uniformly zero)
        return 1

    log2max = np.log2(maxdata)

    iwidth = max(0, int(np.ceil(log2max)))
    if iwidth == int(np.floor(log2max)):  # is a power-of-two integer -> need one extra bit
        iwidth += 1

    if signed:
        # add the sign bit
        iwidth += 1

    return iwidth

# endregion

# region Data type definitions

class NamedType:
    """Class representing a named type.

    For convenience, hls4ml gives names to data types used in the generated HLS. This is equivalent to defining types
    in C/C++ like::

        typedef precision name;

    Args:
        name (str): Name given to the type (used in generated C++/HLS).
        precision (PrecisionType): Precision data type.
    """

    def __init__(self, name, precision, **kwargs):
        self.name = name.format(**kwargs)
        self.precision = precision

class CompressedType(NamedType):
    """Class representing a compressed type in COO format.

    Args:
        name (str): Name given to the type (used in generated C++/HLS).
        precision (PrecisionType): Precision data type.
        index_precision (PrecisionType): Precision of the index of COO format.
    """

    def __init__(self, name, precision, index_precision, **kwargs):
        if not name.startswith('compressed_'):
            name = 'compressed_' + name
        super().__init__(name, precision, **kwargs)
        self.index_precision = index_precision

class ExponentType(NamedType):
    """Special type used to mark an exponent type, used by the power-of-2 quantizers.

    Args:
        name (str): Name given to the type (used in generated C++/HLS).
        precision (PrecisionType): Precision data type.
    """

    def __init__(self, name, precision, **kwargs):
        if not name.startswith('exponent_'):
            name = 'exponent_' + name
        super().__init__(name, precision, **kwargs)
        self.sign = XnorPrecisionType()

class PackedType(NamedType):
    """A type where multiple elements of the tensor are concatenated and stored as a single element, used by the streaming
    implementations to store elements of the last dimension of a tensor as a single element.

    The tensor of shape ``(H, W, C)`` will be represented as a FIFO stream having ``H * W / n_pack`` elements where each
    element will be a concatenation of ``n_elem * n_pack`` elements of the original tensor.

    Args:
        name (str): Name given to the type (used in generated C++/HLS).
        precision (PrecisionType): Precision data type.
        n_elem (int): Number of packed elements.
        n_pack (int): _description_
    """

    def __init__(self, name, precision, n_elem, n_pack, **kwargs):
        super().__init__(name, precision, **kwargs)
        self.n_elem = n_elem
        if n_pack < 0:
            self.n_pack = -n_pack
            self.unpack = True
        else:
            self.n_pack = n_pack
            self.unpack = False

# endregion

# region Variables

class Variable:
    """Base class representing a named multidimensional tensor.

    Args:
        var_name (str): Name of the variable in the generated C++/HLS.
        atype (NamedType): Data type used by the tensor.
    """

    def __init__(self, var_name, atype, **kwargs):
        self.name = var_name.format(**kwargs)
        self.type = atype

class TensorVariable(Variable):
    """Class representing the output of a layer (like an activation tensor).

    Args:
        shape (list, tuple): Shape of the tensor.
        dim_names (list, tuple): Names given to the dimensions of the tensor.
        var_name (str, optional): Name of the variable in the generated C++/HLS. Defaults to ``layer{index}``.
        type_name (str, optional): Name of the data type used (in NamedType). Defaults to ``layer{index}_t``.
        precision (PrecisionType, optional): Precision data type. Defaults to ``None``.
    """

    def __init__(self, shape, dim_names, var_name='layer{index}', type_name='layer{index}_t', precision=None, **kwargs):
        super().__init__(var_name, NamedType(type_name, precision, **kwargs), **kwargs)
        self.shape = shape
        self.dim_names = dim_names

    def get_shape(self):
        return zip(self.dim_names, self.shape)

    def size(self):
        nelem = 1
        for dim in self.shape:
            nelem *= dim
        return nelem

    def size_cpp(self):
        # TODO get rid of size_cpp() (and dim_names)
        return '*'.join([str(k) for k in self.dim_names])

class InplaceTensorVariable(TensorVariable):
    """A ``TensorVariable`` that is just a link to another ``TensorVariable``.

    Args:
        tv (TensorVariable): The tensor variable to link.
        input_var (_type_): The input variable that should be should link to.
    """

    def __init__(self, tv, input_var):
        self.__dict__.update(tv.__dict__)
        self.type = input_var.type
        self.input_var = input_var

class WeightVariable(Variable):
    """Class representing a tensor containing the weights of a layer.

    Precision type of the instance can be modified with the ``update_precision`` method.

    Args:
        var_name (str, optional): Name of the variable in the generated C++/HLS.
        type_name (str, optional): Name of the data type used (in NamedType).
        precision (PrecisionType, optional): Precision data type.
        data (ndarray): The data array.
        quantizer (_type_, optional): Quantizer to apply to the data array. Defaults to ``None``.
    """

    def __init__(self, var_name, type_name, precision, data, quantizer=None, **kwargs):
        super().__init__(var_name, NamedType(type_name, precision, **kwargs), **kwargs)
        self.data = data
        self.nzeros = -1
        self.shape = list(self.data.shape)
        self.data_length = np.prod(self.data.shape)
        self.nonzeros = np.count_nonzero(self.data)
        self.nzeros = self.data_length - self.nonzeros
        self.min = np.min(self.data)
        self.max = np.max(self.data)
        self._iterator = None
        self.update_precision(precision)
        self.quantizer = quantizer

    def __iter__(self):
        self._iterator = np.nditer(self.data, order='C')
        return self

    def __next__(self):
        if not self._iterator.finished:
            value = self._iterator[0]
            self._iterator.iternext()
            return self.precision_fmt.format(value)
        else:
            raise StopIteration

    next = __next__

    def update_precision(self, new_precision):
        self.type.precision = new_precision
        if isinstance(new_precision, (IntegerPrecisionType, XnorPrecisionType, ExponentPrecisionType)):
            self.precision_fmt = '{:.0f}'
        elif isinstance(new_precision, FixedPrecisionType):
            if new_precision.fractional > 0:
                # Use str to represent the float with digits, get the length
                # to right of decimal point
                lsb = 2**-new_precision.fractional
                decimal_spaces = len(str(lsb).split('.')[1])
                self.precision_fmt = f'{{:.{decimal_spaces}f}}'
            else:
                self.precision_fmt = '{:.0f}'
        else:
            raise RuntimeError(f"Unexpected new precision type: {new_precision}")

class CompressedWeightVariable(WeightVariable):
    """Class representing a tensor containing the weights of a layer represented in the COO format.

    Args:
        var_name (str, optional): Name of the variable in the generated C++/HLS.
        type_name (str, optional): Name of the data type used (in NamedType).
        precision (PrecisionType, optional): Precision data type.
        data (ndarray): The data array.
        reuse_factor (_type_): The reuse factor used to pad the data array.
        quantizer (_type_, optional): Quantizer to apply to the data array. Defaults to ``None``.
    """

    def __init__(self, var_name, type_name, precision, data, reuse_factor, quantizer=None, **kwargs):
        super().__init__(var_name, type_name, precision, data, quantizer=quantizer, **kwargs)
        self.extra_zeros = 0
        self.data_length = np.prod(data.shape) - self.nzeros
        while self.data_length % reuse_factor != 0:
            self.extra_zeros += 1
            self.data_length += 1
        self.nonzeros = np.prod(data.shape) - self.nzeros + self.extra_zeros

        # Compress the array
        weights = []
        extra_nzero_cnt = self.extra_zeros
        it = np.nditer(data, order='C', flags=['multi_index'])
        max_idx = 0
        while not it.finished:
            val = it[0]
            if not (val == 0 and extra_nzero_cnt < 1):
                if val == 0:
                    extra_nzero_cnt -= 1
                if it.multi_index[0] > max_idx:
                    max_idx = it.multi_index[0]
                if it.multi_index[1] > max_idx:
                    max_idx = it.multi_index[1]
                weights.append([it.multi_index[1], it.multi_index[0], val])
            it.iternext()
        weights.sort()

        index_precision = 32
        if max_idx > 0:
            index_precision = int(np.log2(max_idx) + 1)
        self.type = CompressedType(type_name, precision, IntegerPrecisionType(width=index_precision, signed=False), **kwargs)

        self.data = weights

    def __iter__(self):
        self._iterator = iter(self.data)
        return self

    def __next__(self):
        value = next(self._iterator)
        value_fmt = self.precision_fmt.format(value[2])
        return f'{{{value[1]}, {value[0]}, {value_fmt}}}'

    next = __next__

class ExponentWeightVariable(WeightVariable):
    """WeightVariable for Exponent aka power-of-2 data. The data should already by quantized by the quantizer.

    Args:
        var_name (str, optional): Name of the variable in the generated C++/HLS.
        type_name (str, optional): Name of the data type used (in NamedType).
        precision (PrecisionType, optional): Precision data type.
        data (ndarray): The data array.
        quantizer (_type_, optional): Quantizer to apply to the data array. Defaults to ``None``.
    """

    def __init__(self, var_name, type_name, precision, data, quantizer=None, **kwargs):
        super().__init__(var_name, type_name, precision, data, quantizer, **kwargs)
        self.type = ExponentType(type_name, precision, **kwargs)
        self.shape = list(self.data.shape[:-1])

    def _format(self):
        y = self.data
        # Use an XnorBinary-like representation for the sign
        sign = np.where(y < 0, np.zeros_like(y), np.ones_like(y))
        # Take the logarithm, since this is what we will write to the header
        # for the optimized product using shifts
        y = (np.log2(np.abs(y)) / np.log2(2.0)).astype('int')
        return np.stack((sign, y), axis=-1)

    def __iter__(self):
        data = self._format()
        self._iterator = iter(data.reshape((np.product(data.shape[:-1]), 2)))
        return self

    def __next__(self):
        value = next(self._iterator)
        value_fmt = self.precision_fmt.format(value[1])
        return f'{{{value[0]}, {value_fmt}}}'

    next = __next__

# endregion

# region Custom source

class Source:
    """Class representing generated source code blocks.

    Args:
        code (str): Generated source code.
    """

    def __init__(self, code):
        self.code = code

    def __str__(self):
        return str(self.code)

# endregion


import argparse
import joblib
import numpy as np
from sklearn.ensemble import RandomForestClassifier

def train_model(x_train, y_train):
    x_train_data = np.load(x_train, allow_pickle=True)
    y_train_data = np.load(y_train, allow_pickle=True)

    model = RandomForestClassifier(n_estimators = 1000, max_depth=10, verbose=1) #depth 4
    model.fit(x_train_data, y_train_data)

    with open('model.pkl', 'wb') as f:
        joblib.dump(model, f)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--x_train')

    parser.add_argument('--y_train')
    args = parser.parse_args()
    train_model(args.x_train, args.y_train)