import keyword
import ast
import re
import codecs
from warnings import warn

from setlx2python.grammar.SetlXgrammarParser import SetlXgrammarParser
from setlx2python.grammar.SetlXgrammarLexer import SetlXgrammarLexer
from setlx2python.grammar.SetlXgrammarListener import SetlXgrammarListener
from antlr4.error.ErrorListener import ErrorListener
from antlr4 import InputStream, CommonTokenStream

from .grammar.types import *  # pylint: disable=unused-wildcard-import
from .state import TranspilerState, ClassContext

class Transpiler:
    """ A class to transpile a SetlX-AST to a Python-AST

    Takes the root element of the SetlX-AST as argument.
    The AST can be transformed with method transpile.

    Attributes
    ----------
    root : Block
        the root element of the SetlX-AST
    state : TranspilerState
        the state of the transpiler
    """

    def __init__(self, root):
        """
        Parameters
        ----------
        root : Block
            the root element of the SetlX-AST
        """
        self.root = root
        self.state = None

    def transpile(self):
        """ Translates the SetlX-AST to an equal Python-AST

        Returns the generated Python-AST
        """
        self.state = TranspilerState()

        body = self.to_python(self.root)
        imports = self.state.imports.to_python(self.state)
        return imports + body

    def to_python(self, node):
        """ Tranlates a given node / node list to a python AST

        The method calls the translation function based on the node's type and returns the result.
        The translation function of a node is the type name in lower case.

        If node is list, every element is translated and the translations are returned as a list in the same order.
        If node is None, None is returned
        """
        if node is None:
            return None
        if isinstance(node, list):
            return [self.to_python(n) for n in node]
        else:
            name = type(node).__name__.lower()
            return getattr(self, name)(*list(node))

    def _prefix_operator(self, operation: str, expr) -> ast.Call:
        """ Turns a prefix operator into a function call

        The function translates the given expression and calls the function *operation*
        with the translated expression as argument.
        e.g. +/[1,2] -> sum([1,2])

        Parameters
        ----------
        operation : str
            The name of the function
        expr :
            The Setlx-Expression that is translated and used as argument for the function
        """
        expr = self.to_python(expr)
        return self.setlx_function(operation, [expr])

    def _binop(self, left, operator, right) -> ast.BinOp:
        """ A helper function to generate infix operations

        The function translates the left and right side of an operation
        and creates a BinOp with the given operation.
        """
        [left, right] = self.to_python([left, right])
        return ast.BinOp(left, operator, right)

    def _compare(self, left, operator, right) -> ast.Compare:
        """ A helper function to generate comparisions

        The function translates the left and right side of an operation
        and creates a Compare ast operation with the given operator.
        """
        [left, right] = self.to_python([left, right])
        return ast.Compare(left=left, ops=[operator], comparators=[right])

    def _boolop(self, left, operator, right) -> ast.BoolOp:
        """ A helper function to generate the ast for boolean operators

        The function translates the left and right side of an operation
        and creates a BoolOp ast operation with the given operator.
        """
        left = self.to_python(left)
        right = self.to_python(right)
        return ast.BoolOp(op=operator, values=[left, right])

    def _augassign(self, assignable, operator, right_hand_side) -> ast.AugAssign:
        """ A helper function to generate augmented assignments (e.g +=, -=)

        The function translates the assignables and right hand side of an augmented assignment
        and creates a AugAssign ast object with the given operator.
        """
        [target, rhs] = self.to_python([assignable, right_hand_side])
        return ast.AugAssign(target=target, op=operator, value=rhs)

    def assignablecollectionaccess(self, assignable, exprs) -> ast.Subscript:
        """ Translates a AssignableCollectionAccess object

        A AssignableCollectionAccess is an assignment to the key of a object (e.g. a_list[2] = "test")
        It is translates to a python Subscript object, which is the corresponding python lexical element.
        e.g. a_list[2] -> a_list[2]
        """
        assignable = self.to_python(assignable)
        py_exprs = self.to_python(exprs)
        # unpack collections with only one memeber
        index = ast.List(elts=py_exprs) if len(exprs) > 1 else py_exprs[0]
        return ast.Subscript(value=assignable, slice=ast.Index(index))

    def assignableignore(self) -> ast.Name:
        """ Translates an assignment to a variable which is ignored

        In python the variable "_" is used to ignore an assignment.
        So we translate this to ast.Name(id="_").
        """
        return ast.Name(id="_")

    def assignablelist(self, assignables: list) -> ast.List:
        """ Translates an assigment to a list of variables

        A list in python ca be unpacked by writing::

            [var1, var2, var3] := a_list;

        The same is possible in python so the element is just translated
        to a list with the variables as elements.

        """
        assignables = [self.to_python(a) for a in assignables]
        return ast.List(elts=assignables)

    def assignablemember(self, assignable, member) -> ast.Attribute:
        """ Translates the assignment to the member of an object

        Attributes of objects can be assigned by writing::

            object.attribute := value;

        This notation is the same in python so it can be translated
        to an ast.Attribute object.
        """
        [target, member] = self.to_python([assignable, member])
        return ast.Attribute(value=target, attr=member)

    def assignablevariable(self, id: str):
        """ Translates a variable which is on the right hand side of an assignment

        By default this is translated to a python variable (ast.Name),
        but there are two special cases:

        1.  If the variable is called "this" we need to translate it to the corresponding keyword which is "self".

        2.  If the transpiler is in a class and the variable is a class attribute
            the variable needs to be prefixed with "self." to enable the attribute access in pythons

        """
        if id == "this":
            return ast.Name(id="self")

        # prefix variable with "v_" if the id is a python keyword
        py_id = escape_id(id)

        if self.state.is_class_variable(py_id) and py_id not in self.state.variables and py_id not in self.state.procedures:
            return ast.Attribute(value=ast.Name(id="self"), attr=py_id)
        else:
            self.state.variables.append(py_id)

        return ast.Name(id=py_id)

    def assignment(self, assignables, right_hand_side) -> ast.Assign:
        """ Translates an assignment to a python assignment

        If the right hand side is a single variable,
        it is wraped into a copy statement, since setlx is value based
        and this behaviour needs to be mimicked in python.
        """
        left = self.to_python(assignables)
        right = self.to_python(right_hand_side)

        if isinstance(right_hand_side, Variable):
            right = self.setlx_function("copy", [right])

        # unpack function calls (calls are not wraped in expressions in python)
        if isinstance(right, ast.Expr) and isinstance(right.value, ast.Call):
            right = right.value

        return ast.Assign(targets=left, value=right)

    def backtrack(self) -> ast.Raise:
        """ Translates a backtrack statement to a python BacktrackException raise"""
        return ast.Raise(
            exc=self.setlx_function("BacktrackException", []),
            cause=None
        )

    def block(self, stmnts: list) -> list:
        """ converts a list of setlx statements to a list of python statements

        This function converts a list of setlx statements line by line.
        """
        # copy state variables to restore them after block translation
        variables = self.state.variables[:]
        self.state.variables = []
        built_ins = self.state.built_ins[:]
        procedures = {**self.state.procedures}

        # count number of procedures in a block for non ambigous function naming
        procedure_counter = self.state.procedure_counter
        self.state.procedure_counter = 0

        py_stmnts = []
        for s in stmnts:
            level = self.state.level
            self.state.level += 1  # next depth level
            self.state.add_before = lambda stmnt: py_stmnts.append(stmnt)

            stmnt = self.to_python(s)

            self.state.add_before = None
            self.state.level = level

            # certain elements need to be warped into an expression
            if isinstance(stmnt, (ast.Compare, ast.UnaryOp, ast.BinOp, ast.Call, ast.Set, ast.SetComp)):
                stmnt = ast.Expr(value=stmnt)
            py_stmnts.append(stmnt)

        self.state.procedure_counter = procedure_counter

        self.state.variables = variables
        self.state.built_ins = built_ins
        self.state.procedures = procedures
        return py_stmnts

    def _bool_op(self, left, operator, right) -> ast.Compare:
        """ Helper function that generates a Compare element and wraps the left and right side into the bool function """
        [left_cond, right_cond] = self.to_python([left, right])
        left = call_function("bool", [left_cond])
        right = call_function("bool", [right_cond])
        return ast.Compare(left=left, ops=[operator], comparators=[right])

    def booleanequal(self, left, right) -> ast.BoolOp:
        """ x <==> y -> bool(x) == bool(y) """
        return self._bool_op(left, ast.Eq(), right)

    def booleannotequal(self, left, right) -> ast.BoolOp:
        """ x <!=> y -> bool(x) != bool(y) """
        return self._bool_op(left, ast.NotEq(), right)

    def cachedprocedure(self, params, block) -> ast.Name:
        """ See method proceduredefinition(...) """
        proc_name = f"procedure_{self.state.level}_{self.state.procedure_counter}"

        self.state.add_before(self.proceduredefinition(
            CachedProcedure(params, block), proc_name))

        return ast.Name(id=proc_name)

    def cardinality(self, expr) -> ast.Call:
        """ Python equivilant is len function

        e.g. #A -> len(A)
        """
        expr = self.to_python(expr)
        return call_function("len", [expr])

    def cartesianproduct(self, left, right) -> ast.Call:
        """ Translated with setlx.cartesian_product function

        e.g. x >< y -> setlx.cartesian_product(x,y)
        """
        params = self.to_python([left, right])
        return self.setlx_function("cartesian_product", params)

    def check(self, check_block, backtrack_block) -> ast.Try:
        """ Check is translated to a try-catch which catches BacktrackExceptions and does nothing on exception

        e.g.::

            check{
                ...
            }

        is translated to::

            try:
                ..
            except BacktrackException: pass
        """
        body = self.to_python(check_block)
        afterBacktrack = [ast.Pass()]
        if backtrack_block != None:
            afterBacktrack = self.to_python(backtrack_block)
        return ast.Try(body=body,
                       handlers=[
                           ast.ExceptHandler(type=self.setlx_access('BacktrackException'),
                                             name=None,
                                             body=afterBacktrack)
                       ],
                       orelse=[],
                       finalbody=[])

    def classconstructor(self, id: str, params: list, block: Block, static_block: Block) -> ast.ClassDef:
        """ Creates the code for a python class based on a setlx class

        This class is inherits from setlx.SetlXClass.
        The class body from setlx is packed into the __init__ function of the class.
        The __init__ functions gets the parameters of the class.
        The content of the static block is translated into the class body directly.
        In the __init__ functions all static functions are converted into methods by using the function setlx.to_method(..)

        e.g.::

            class Test(x){
                this.x = x;
                static{
                    name := "Test";
                    y := procedure(a,b){
                        ...
                    };
                }
            }

        is translated to::

            class Test(setlx.SetlXClass):
                name = "Test";
                @staticmethod
                def y(a,b, self=None):
                    ...

                def __init__(self,x):
                    [x] = setlx.copy([x])
                    self.x = x
                    self.y = setlx.to_method(self,Test.y)
        """
        static_block = static_block or Block([])  # convert None to empty list

        py_id = escape_id(id)
        self.state.check_built_ins(py_id)

        self.state.class_context = ClassContext(py_id)
        self.state.class_context.variables = [escape_id(
            s.name) for s in block.stmnts+static_block.stmnts if isinstance(s, (ProcedureDefinition, LambdaDefinition))]

        for s in block.stmnts:
            if isinstance(s, Assignment):
                self.state.class_context.variables.append(
                    escape_id(s.assignables[0].id))

        procedure = Procedure(params, block)
        init = self.proceduredefinition(procedure, "__init__")
        body = init.body  # pylint: disable=no-member

        for i, s in enumerate(body):
            if isinstance(s, ast.Assign) and isinstance(s.targets[0], ast.Name):
                # prefix all variable assignments with self.*
                # e.g. test = 2
                #   => self.test = 2
                if isinstance(s.targets[0], ast.Name):
                    s.targets[0] = s.targets[0].id
                self.state.class_context.variables.append(s.targets[0])
                s.targets = [
                    ast.Attribute(
                        value=ast.Name(id='self'),
                        attr=s.targets[0]
                    ),
                    ast.Name(id=s.targets[0])
                ]
            # enumerate over all generated functions and assign them as class method
            # e.g. self.method = setlx.to_method(self,method)
            if isinstance(s, ast.FunctionDef):
                assign = self.to_method(s.name, False)
                self.state.class_context.variables.append(s.name)
                body.insert(i+1, assign)

        # translate static part of class
        self.state.class_context.static = True
        static = self.to_python(static_block)
        self.state.class_context.static = False

        # in setlx static functions can be called as method
        # in order to translate this properly, every static function is converted to a method in the constructor
        # looks like this: self.static_function = setlx.to_method(self,static_function)
        for s in static:
            if isinstance(s, ast.FunctionDef):
                assign = self.to_method(s.name, True)
                body.insert(0, assign)
            # check for all static lambda definitions and make them none static
            elif isinstance(s, ast.Assign) and isinstance(s.value, ast.Lambda):
                assign = self.to_method(s.targets[0].id, True)
                body.insert(0, assign)

        self.state.class_context = None
        return ast.ClassDef(
            name=py_id,
            bases=[self.setlx_access("SetlXClass")],
            keywords=[],
            body=static+[init],
            decorator_list=[]
        )

    def closure(self, params, block) -> ast.Name:
        """ Closures are translated like normal procedures (see procedure(..))"""
        warn("closures are translated as normal python functions, so they can only read from variables outside the scope")
        proc_name = f"closure_{self.state.level}_{self.state.procedure_counter}"

        self.state.add_before(self.proceduredefinition(
            Closure(params, block), proc_name))

        return ast.Name(id=proc_name)

    def collectionaccess(self, params, callable) -> ast.Subscript:
        """ Tranalated to Python Subscript

        Three possible params described by the following examples:
        e.g.    collection[1] -> collection[1]
                collection[1,2] -> collection[[1,2]]
                collection[1..2] -> collection[1:2]
        """
        py_callable = self.to_python(callable)

        if isinstance(params, list):
            py_params = ast.List(elts=self.to_python(params))
        else:
            py_params = self.to_python(params)

        if isinstance(params, ListRange):
            return ast.Subscript(value=py_callable, slice=py_params)
        else:
            index = ast.Index(value=py_params)
            return ast.Subscript(value=py_callable, slice=index)

    def collectmap(self, expr, callable) -> ast.Call:
        """ Translated by calling the method "collect" on the object

        e.g. collection{1} -> collection.collect(1)
        """
        [py_var, py_expr] = self.to_python([callable, expr])
        return ast.Call(func=ast.Attribute(value=py_var, attr='collect'), args=[py_expr], keywords=[])

    def condition(self, expression):
        """ Translated by translating the cxondition's expression """
        return self.to_python(expression)

    def conjunction(self, left, right) -> ast.BoolOp:
        """ Same in SetlX and Python (and, &&) """
        return self._boolop(left, ast.And(), right)

    def difference(self, left, right) -> ast.BinOp:
        """ Same in SetlX and Python (-) """
        return self._binop(left, ast.Sub(), right)

    def differenceassignment(self, assignable, right_hand_side) -> ast.AugAssign:
        """ Same in SetlX and Python (-=) """
        return self._augassign(assignable, ast.Sub(), right_hand_side)

    def disjunction(self, left, right) -> ast.BoolOp:
        """ Same in SetlX and Python (or, ||) """
        return self._boolop(left, ast.Or(), right)

    def dowhile(self, condition, block) -> ast.While:
        """ Translated by converting to Python while structure with condition at end

        e.g.::

            do{
                ...
            }while(x > 10);

        is transalted to::

            while True:
                ...
                if not(x > 10):
                    break
        """
        [py_condition, py_block] = self.to_python([condition, block])
        break_block = [ast.Break()]
        negate = ast.UnaryOp(op=ast.Not(), operand=py_condition)
        if_break = ast.If(test=negate, body=break_block, orelse=[])
        py_block.append(if_break)
        return ast.While(test=bool_true(), body=py_block, orelse=[])

    def equal(self, left, right) -> ast.Compare:
        """ Same in SetlX and Python (==)"""
        return self._compare(left, ast.Eq(), right)

    def exists(self, iter_chain, condition) -> ast.Call:
        """ Translated with Python's any function

        e.g. exists(x in y | x > 2) -> any(x > 2 for x in y)
        """
        expr = self.to_python(SetlIteration(condition, iter_chain, None))
        expr = expr.args[0]  # unpack list from setlx.List object
        return call_function("any", [ast.GeneratorExp(elt=expr.elt, generators=expr.generators)])

    def exit(self):
        """ Translated by calling python's exit function"""
        return call_function("exit", [])

    def explicitlist(self, exprs) -> ast.Call:
        """ Translated by using custom setlx.List

        e.g. [1,2,3] -> seltx.List([1,2,3])
        """
        list_arg = ast.List(elts=self.to_python(exprs))
        return self.setlx_function("List", [list_arg])

    def explicitlistwithrest(self, exprs, rest) -> ast.Call:
        """ Translated by adding the rest as starred element to the list

        e.g. [1,2|x] -> setlx.List([1,2,*x])
        """
        py_exprs = self.to_python(exprs)
        py_exprs.append(ast.Starred(
            value=self.to_python(rest)
        ))
        elts = ast.List(elts=py_exprs)
        return self.setlx_function("List", [elts])

    def factorial(self, expr) -> ast.Call:
        """ Translated by using the setlx.factorial function

        e.g. 5! -> setlx.factorial(5)
        """
        expr = self.to_python(expr)
        return self.setlx_function("factorial", [expr])

    def forall(self, iter_chain, condition) -> ast.Call:
        """ Translated with Python all function

        e.g. forall(x in y | x > 2) -> all(x > 2 for x in y)
        """
        expr = self.to_python(SetlIteration(condition, iter_chain, None))
        expr = expr.args[0]  # unpack list from setlx.List object
        return call_function("all", [ast.GeneratorExp(elt=expr.elt, generators=expr.generators)])

    def functioncall(self, params, callable) -> ast.Call:
        """ Function calls in SetlX and Python are the same so they are translated equally

        There two speciall functions:
        1. "load" is translated to a import statement (load("test.stlx") -> from test import *)
        2. "eval" gets the global and local variables as extra parameters (eval(x) -> setlx.eval(x,globals(),locals()))
        """
        py_params = self.to_python(params)

        if isinstance(callable, Variable):
            if callable.id == "load":
                return import_call(py_params[0].s)
            elif callable.id == "eval":
                return self.setlx_function("eval", [py_params[0],
                                                    ast.Call(func=ast.Name(id='globals'),
                                                             args=[], keywords=[]),
                                                    ast.Call(func=ast.Name(id='locals'), args=[], keywords=[])])

        expr = self.to_python(callable)
        return ast.Call(expr, py_params, [])

    def greaterorequal(self, left, right) -> ast.Compare:
        """ same in SetlX and Python (>=) """
        return self._compare(left, ast.GtE(), right)

    def greaterthan(self, left, right) -> ast.Compare:
        """ same in SetlX and Python (>) """
        return self._compare(left, ast.Gt(), right)

    def ifthen(self, condition, block, else_list) -> ast.If:
        """ Translated to a Python If statement, which has the same structure as in SetlX """
        [condition, block] = self.to_python([condition, block])

        orelse = []

        for e in else_list[:: -1]:
            if isinstance(e, IfThenBranch):
                if isinstance(orelse, list):
                    if len(orelse) > 0:
                        e.orelse.append(orelse[0])
                else:
                    e.orelse.append(orelse)
                orelse = [e]
            else:
                orelse = e

        orelse = self.to_python(orelse) if isinstance(orelse, Block) else [
            self.to_python(e) for e in orelse]
        return ast.If(test=condition, body=block, orelse=orelse)

    def ifthenbranch(self, condition, block, orelse) -> ast.If:
        """ Translated to a Python If statement, which has the same structure as in SetlX """
        [condition, block] = self.to_python([condition, block])
        if len(orelse) > 0:
            if isinstance(orelse[0], Block):
                else_list = self.to_python(orelse[0])
            else:
                else_list = [self.to_python(e) for e in orelse]
        else:
            else_list = []

        return ast.If(test=condition, body=block, orelse=else_list)

    def implication(self, left, right) -> ast.Compare:
        """ "x => y" is translated to "not x or y" """
        [left, right] = self.to_python([left, right])
        not_left = ast.UnaryOp(op=ast.Not(), operand=left)
        return ast.Compare(left=not_left, ops=[ast.Or()], comparators=[right])

    def integerdivision(self, left, right) -> ast.BinOp:
        """ same in SetlX and Python (\\) """
        return self._binop(left, ast.FloorDiv(), right)

    def integerdivisionassignment(self, assignable, right_hand_side) -> ast.AugAssign:
        """ same in SetlX and Python (\\=) """
        return self._augassign(assignable, ast.FloorDiv(), right_hand_side)

    def iterator_from_chain(self, iter_chain: list):
        """ This function converts a list of iterators.

        If the iterator chain only consists of one iterator, it can be converted to a python generator.
        The function returns this as a python expression.
        e.g.::

            x*2 : x in a

        is translated to::

            x*2 for x in a

        In setlX iterators can be chained together like so::

            [x,y]: x in a, y in b

        this is translated via the function setlx.cartesian_product which creates a possible combinations::

            [x,y] for (x,y) in setlx.cartesian_product(a,b)

        """
        if len(iter_chain) == 1:
            return self.to_python([iter_chain[0].assignable, iter_chain[0].expression])

        assignables = [self.to_python(i.assignable) for i in iter_chain]
        assignable = ast.List(elts=assignables)

        exprs = [self.to_python(e.expression) for e in iter_chain]
        list_product = self.setlx_function("cartesian_product", exprs)

        return [assignable, list_product]

    def lambdaclosure(self, params, expr) -> ast.Lambda:
        """ Translated like LambdaProcedure (see lambdaprocedure method) """
        return self.lambdaprocedure(params, expr)

    def lambdadefinition(self, procedure, name: str) -> ast.Assign:
        """ Translated to assignment of lambda function

        e.g.::

            x := a,b |-> a*b;

        is translated to::

            x = lambda a,b: a*b
        """
        py_id = escape_id(name)
        lambda_py = self.lambdaprocedure(procedure.params, procedure.expr)
        return ast.Assign(targets=[ast.Name(id=py_id)], value=lambda_py)

    def lambdaprocedure(self, params: list, expr) -> ast.Lambda:
        """ Translated to Python Lambda

        If the lambda is in the static block of a class, the parameter "self=None"
        is added to support a call as class function.
        """
        expr = self.to_python(expr)
        params = self.to_python(params)

        defaults = []
        if self.state.is_class_static():
            # add self=None as parameter in static class block
            params.append(ast.arg(arg="self", annotation=None))
            defaults.append(ast.NameConstant(None))

        return ast.Lambda(args=ast.arguments(args=params,
                                             vararg=None,
                                             kwonlyargs=[],
                                             kw_defaults=[],
                                             kwarg=None,
                                             defaults=defaults),
                          body=expr)

    def lessorequal(self, left, right) -> ast.Compare:
        """ Same in SetlX and Python (<=)"""
        return self._compare(left, ast.LtE(), right)

    def lessthan(self, left, right) -> ast.Compare:
        """ Same in SetlX and Python (<)"""
        return self._compare(left, ast.Lt(), right)

    def listparameter(self, id):
        raise "not reachable"

    def listrange(self, start, end) -> ast.Slice:
        """ same in SetlX and Python """
        [start, end] = self.to_python([start, end])
        return ast.Slice(lower=start, upper=end, step=None)

    def match(self):
        raise NotSupported("match is not supported")

    def memberaccess(self, parent, child) -> ast.Attribute:
        """ Translated to ast.Attribute (same in SetlX and Python)"""
        [parent, child] = self.to_python([parent, child])
        if isinstance(child, ast.Name):
            # unpack id to generate correct python ast
            child = child.id
        return ast.Attribute(value=parent, attr=child)

    def minus(self, expr) -> ast.UnaryOp:
        """ Same in SetlX and Python (-)"""
        expr = self.to_python(expr)
        return ast.UnaryOp(op=ast.USub(), operand=expr)

    def modulo(self, left, right) -> ast.BinOp:
        """ Same in SetlX and Python (%)"""
        return self._binop(left, ast.Mod(), right)

    def moduloassignment(self, assignable, right_hand_side) -> ast.AugAssign:
        """ Same as in python (%=) """
        return self._augassign(assignable, ast.Mod(), right_hand_side)

    def notequal(self, left, right) -> ast.Compare:
        """ Translated to python "!=" """
        return self._compare(left, ast.NotEq(), right)

    def notin(self, left, right) -> ast.Compare:
        """ Translated to python "not in" """
        return self._compare(left, ast.NotIn(), right)

    def operatorexpression(self, expr) -> ast.Starred:
        """ Translated to starred argument """
        expr = self.to_python(expr)
        return ast.Starred(value=expr)

    def parameter(self, id, default) -> ast.arg:
        """ Translated to ast.arg.

        The default parameter is not used since in python
        it is not stored in the argument but in a seperated list.
        """
        py_id = escape_id(id.id)
        if py_id in self.state.built_ins:
            del self.state.built_ins[self.state.built_ins.index(py_id)]
        self.state.variables.append(py_id)
        return ast.arg(arg=py_id, annotation=None)

    def power(self, base, exponent) -> ast.BinOp:
        """ "**" operator is same in python """
        return self._binop(base, ast.Pow(), exponent)

    def prefixoperator(self, expr, py_target_type) -> ast.Call:
        """ Helper function for generating a function call """
        expr = self.to_python(expr)
        return call_function(py_target_type, [expr])

    def procedure(self, params, block) -> ast.Name:
        """ Translated by defining a function in the statement before and using the functions name as translation

        e.g.::

            x := [procedure(a,b){...}]

        is translated to::

            def procedure_0_0(a,b):
                ...
            x = [procedure_0_0]
        """
        proc_name = f"procedure_{self.state.level}_{self.state.procedure_counter}"

        self.state.add_before(self.proceduredefinition(
            Procedure(params, block), proc_name))

        return ast.Name(id=proc_name)

    def proceduredefinition(self, procedure, name: str) -> ast.FunctionDef:
        """ This function creates a function definition with the given name and the content of the given procedure

        The function has the same params and block like the procedure.
        If it is a class method, the parameter "self" is added as first parameter.
        If it is a static class function the decorator "staticmethod" is used.
        If the function is a cached procedure the decorator setlx.cached_procedure is used to mimic this functionality.
        At the beginning of the function all parameters that are not read-write-parameters are copied in order to make the
        functions call by value.
        e.g.::

            test := cachedProcedure(x,y, rw A){
                ..
            };

        is translated to::

            @setlx.cached_procedure
            def test(x,y,A: 'rw'):
                [x,y] = setlx.copy([x,y])
                ...

        """
        py_name = escape_id(name)
        self.state.procedure_counter += 1

        [params, block] = procedure

        py_params = []
        defaults = []  # default values for parameters
        vararg = None
        value_params = []
        for p in params:
            if not isinstance(p, ListParameter):
                prm = self.to_python(p)
                if not isinstance(p, ReadWriteParameter):  # only copy value parameters
                    value_params.append(prm.arg)
                    if p.default != None:  # add default value if one is given
                        defaults.append(self.to_python(p.default))
                py_params.append(prm)
            else:
                py_id = self.to_python(p.id)
                value_params.append(py_id.id)
                vararg = ast.arg(arg=py_id.id, annotation=None)

        decorators = [self.setlx_access("cached_procedure")] if isinstance(
            procedure, CachedProcedure) else []

        if self.state.is_class_static():
            # if function is static, add self=None as optinal parameter to the end
            py_params.append(ast.arg(arg="self", annotation=None))
            defaults.append(ast.NameConstant(value=None))
            decorators.insert(0, ast.Name(id="staticmethod"))
        elif self.state.class_context != None:
            # add self.function_name = function name to make it a method
            self.state.class_context.variables.append(py_name)
            py_params.insert(0, ast.arg(arg="self", annotation=None))

        arguments = ast.arguments(args=py_params,
                                  vararg=vararg,
                                  kwonlyargs=[],
                                  kw_defaults=[],
                                  kwarg=None,
                                  defaults=defaults)

        block = self.to_python(block)
        if len(value_params) > 0:
            # deep copy parameters to allow call by value
            block.insert(0, self.copy_params(value_params))

        self.state.check_built_ins(py_name)
        self.state.procedures[py_name] = params
        return ast.FunctionDef(name=py_name, args=arguments, body=block, decorator_list=decorators, returns=None)

    def product(self, left, right) -> ast.BinOp:
        """ same in python """
        return self._binop(left, ast.Mult(), right)

    def productassignment(self, assignable, right_hand_side) -> ast.AugAssign:
        """ "*=" is same in python """
        return self._augassign(assignable, ast.Mult(), right_hand_side)

    def productofmembers(self, expr) -> ast.Call:
        """ Translated by calling the function product with the expression as argument

        e.g. */[1,2] -> product([1,2])
        """
        return self._prefix_operator("product", expr)

    def productofmembersbinary(self, left, right) -> ast.Call:
        """ Translated by calling the function setlx.product with the left and right side as arguments

        e.g. 2 */[1,2] -> setlx.product([1,2], 2)
        """
        [left, right] = self.to_python([left, right])
        return self.setlx_function("product", [right, left])

    def quotient(self, left, right) -> ast.BinOp:
        """ "/"-perator is same in python """
        return self._binop(left, ast.Div(), right)

    def quotientassignment(self, assignable, right_hand_side) -> ast.AugAssign:
        """ "/=" is same in python """
        return self._augassign(assignable, ast.Div(), right_hand_side)

    def range(self, start, step, end) -> ast.Call:
        """ translated with setlx._range function

        Step is calculated by substrating the second from the first expression.
        e.g. [1,3..10] -> setlx._range(1, 10, 3-1)
        """
        [start, end] = self.to_python([start, end])
        range_params = [start, end]

        if step != None:
            # calculate step by substracting step from start
            py_step = ast.BinOp(left=self.to_python(
                step), op=ast.Sub(), right=start)
            range_params.append(py_step)

        return self.setlx_function("_range", range_params)

    def readwriteparameter(self, id) -> ast.arg:
        """ Translated as normal arugment (ast.arg) with "rw" annotation """
        py_id = self.to_python(id)
        return ast.arg(arg=py_id.id, annotation=ast.Str("rw"))

    def scan(self):
        raise NotSupported("scan is not supported")

    def setlistconstructor(self, collection) -> ast.Call:
        """ Translated by calling setlx.Set(...) with collection as argument

        e.g. {1,2,3} -> setlx.Set([1,2,3])
        """
        elts = []
        if collection == None:
            elts = []
        else:
            py_collection = self.to_python(collection)
            if isinstance(collection, Range):
                elts = [py_collection]
            else:
                elts = [py_collection.args[0]]
        return self.setlx_function("Set", elts)

    def setliteration(self, expr, iter_chain, condition) -> ast.Call:
        """ Translated to setlx.List with generator as argument"""
        iter_chain = self.to_python(iter_chain)
        expr = self.to_python(expr)
        condition = [self.to_python(condition)] if condition != None else []

        iter_chain[-1].ifs = condition
        return self.setlx_function("List", [ast.ListComp(elt=expr, generators=iter_chain)])

    def setliterator(self, assignable, expression) -> ast.comprehension:
        """ Translated with python equivalent which is a comprehension"""
        [assignable, expression] = self.to_python([assignable, expression])
        return ast.comprehension(target=assignable, iter=expression, ifs=[], is_async=0)

    def setlvector(self, values):
        """ Translated by calling setlx.Vector constructor with values as argument

        e.g. << 1 2 >> -> setlx.Vector([1,2])
        """
        py_values = self.to_python(values)
        return self.setlx_function("Vector", py_values)

    def setlmatrix(self, vectors):
        """ Translated by calling setlx.Matrix constructor with vectors as argument

        e.g. << << 1 2 >> << 2 1 >> >> -> setlx.Matrix([[1,2],[2,1]])
        """
        py_vec = [ast.List(elts=self.to_python(v.values)) for v in vectors]
        return self.setlx_function("Matrix", [ast.List(elts=py_vec)])

    def setlxassert(self, condition, expr) -> ast.Assert:
        """ Translated to Python Assert statement """
        [condition, expr] = self.to_python([condition, expr])
        return ast.Assert(test=condition, msg=expr)

    def setlxbreak(self) -> ast.Break:
        """ Translated to Python Break statement """
        return ast.Break()

    def setlxcontinue(self) -> ast.Continue:
        """ Translated to Python Continue statement """
        return ast.Continue()

    def setlxdouble(self, value: str) -> ast.Num:
        """ Translated as a Python number """
        return ast.Num(n=float(value))

    def setlxfalse(self) -> ast.NameConstant:
        """ Translates "true" to "True" """
        return bool_false()

    def setlxfor(self, iteratorChain, condition, block):
        """ Translates SetlX For to Python For

        Python does not support a conditions in for loops
        so they are translated as seperate if statement in loop.
        e.g.::

            for(x in list | x > 2){...}

        translated to::

            for x in list:
                if x > 2:
                    ...
        """
        block = self.to_python(block)
        [assignable, iterator] = self.iterator_from_chain(iteratorChain)
        # add condition as if self.statement in branch
        if condition != None:
            condition = self.to_python(condition)
            if_stmt = ast.If(test=condition, body=block, orelse=[])
            block = [if_stmt]
        return ast.For(target=assignable, iter=iterator, body=block, orelse=[])

    def setlxfraction(self, number: str) -> ast.Num:
        """ Translated as a Python number """
        return ast.Num(n=int(number))

    def setlxin(self, left, right) -> ast.Compare:
        """ Translated with Python in keyword """
        return self._compare(left, ast.In(), right)

    def setlxlist(self, cb) -> ast.Call:
        """ Translated by calling setlx.List(..) with list as argument

        e.g. [1,2,3] -> setlx.List([1,2,3])
        """
        elts = []
        if cb != None:
            elts = [self.to_python(cb)]
            if isinstance(cb, (ExplicitList, ExplicitListWithRest, SetlIteration)):
                return elts[0]
        return self.setlx_function("List", elts)

    def setlxliteral(self, value: str) -> ast.Str:
        """ Translated with Python string, stripping the "'" at start and beginning """
        # cut " from start and end
        return ast.Str(s=value[1: -1])

    def setlxnot(self, expr) -> ast.UnaryOp:
        """ Translated with Python not keyword """
        expr = self.to_python(expr)
        return ast.UnaryOp(op=ast.Not(), operand=expr)

    def setlxom(self) -> ast.NameConstant:
        """ Translated with Python None """
        return ast.NameConstant(None)

    def setlxreturn(self, expression) -> ast.Return:
        """ Translated with Python return statement """
        expression = self.to_python(expression)
        return ast.Return(value=expression, decorator_list=[], returns=None)

    def setlxstring(self, string: str) -> ast.Str:
        """ This function translates setlx to python strings

        This can be quite complicated due to the fact that setlx strings can include expressions.
        In order to support this in python f-strings are beeing used.
        Since the expressions are not recognized by the parser, we need to recognize the expressions in here.
        e.g.::

            "$x$ + $y$ = $x+y$"

        is translated to::

            f'{x} + {y} = {x+y}'

        """
        value = string[1:-1]
        if len(value) == 0:
            # if the string is empty, so is the python string
            return ast.Str(s="")
        value = value.replace("\\n", "\n").replace(
            "\\t", "\t").replace("\\\"", "\"").replace("\\'", "s")
        # split string by expressions in string
        # e.g. "test $x$ = $x$" => ["test","$x$"," = ","$x$"]
        matches = re.finditer(r'\$(?!\\)[^$]+\$', value)
        indices = []
        for i in matches:
            indices += i.span()

        if len(indices) == 0:
            return ast.Str(s=value)

        if indices[0] != 0:
            indices.insert(0, 0)

        slices = [value[i:j] for i, j in zip(
            indices, indices[1:]+[None]) if len(value[i:j]) > 0]

        values = []
        for s in slices:
            if s[0] == "$":
                newline = "\n"
                tab = "\t"
                if len(s) == 1 or s[-1] != "$":
                    raise SyntaxError(
                        f'Error(s) while parsing string "{value}" {"{"}{newline}{tab}{"$ missing"}{newline}{"}"}')

                # escape backslashes
                escaped = codecs.unicode_escape_decode(s.strip("$"))[0]
                try:
                    setlx_expr = parse_expr(escaped)
                except Exception as e:
                    raise SyntaxError(
                        f'Error(s) while parsing string "{value}" {"{"}{newline}{tab}{e}{newline}{"}"}')
                expr = self.to_python(setlx_expr)
                values.append(
                    ast.FormattedValue(
                        value=expr, conversion=-1,
                        format_spec=None)
                )
            else:
                values.append(ast.Str(s))
        if len(values) == 1 and isinstance(values[0], str):
            return ast.Str(s=values[0])
        else:
            return ast.JoinedStr(values=values)

    def setlxtrue(self):
        """ Translates "true" to "True" """
        return bool_true()

    def setlxwhile(self, condition, block):
        """ Translates while construct to python while """
        [py_condition, py_block] = self.to_python([condition, block])
        return ast.While(test=py_condition, body=py_block, orelse=[])

    def sum(self, left, right):
        """ Translates SetlX plus operator with python plus operator """
        return self._binop(left, ast.Add(), right)

    def sumassignment(self, assignable, right_hand_side) -> ast.AugAssign:
        """ Translates += statement with python += statement """
        return self._augassign(assignable, ast.Add(), right_hand_side)

    def sumofmembers(self, expr) -> ast.Call:
        """ Translated by calling the function sum with the expression as argument"""
        return self._prefix_operator("sum", expr)

    def sumofmembersbinary(self, left, right) -> ast.Call:
        """ Translated by calling the function setlx.sum with the left and right side as arguments"""
        [left, right] = self.to_python([left, right])
        return self.setlx_function("sum", [right, left])

    def switch(self, case_list: list, default_branch) -> ast.If:
        """ Switch is translated to a list of if statements

        A switch statement is translated to a list of if statements
        where every value is checked if "elif" and the the default branch
        is represented by an else statement.

        If there only is a default branch no if is needed so the
        translation of the default branch is returned
        """
        if len(case_list) == 0:
            # in setlx try can standalone, in python not
            return self.to_python(default_branch) or Block([])

        cond = self.to_python(case_list[0][0])
        blk = self.to_python(case_list[0][1])
        orelse = self.to_python(default_branch) or []

        for e in case_list[:: -2]:  # invert list and cut first element
            [test, block] = self.to_python(list(e[0:2]))
            orelse = [ast.If(test=test, body=block, orelse=orelse)]

        return ast.If(test=cond, body=blk, orelse=orelse)

    def term(self):
        raise NotSupported("terms are not supported")

    def trycatch(self, block: list, try_list: list) -> ast.Try:
        """ Translates a try catch construct

        The translation is best described by an example::

            try{
                throw("error");
            } catchUsr(e){
                print("user error: $e$")
            } catchLng(e){
                print("language error: $e$")
            } catch(e){
                print("error: $e$")
            }

        This is translated to::

            try:
                setlx.throw("error");
            except setlx.UserExcetion as e:
                e = setlx.unpack_error(e)
                setlx.print(f"user error: {e}")
            except Exception as e:
                e = setlx.unpack_error(e)
                setlx.print(f"user error: {e}")
            except Exception as e:
                e = setlx.unpack_error(e)
                setlx.print(f"error: {e}")
        """
        block = self.to_python(block)
        if len(try_list) == 0:
            return block

        handlers = []
        for b in try_list:
            if isinstance(b, TryCatchUsrBranch):
                error_type = self.setlx_access("UserException")
            else:
                error_type = ast.Name(id="Exception")

            var = self.to_python(b.variable)
            error = self.unpack_error(var)
            except_block = [error] + self.to_python(b.block)
            ex = ast.ExceptHandler(
                type=error_type,
                name="e",
                body=except_block
            )
            handlers.append(ex)

        return ast.Try(
            body=block,
            handlers=handlers,
            orelse=[],
            finalbody=[]
        )

    def trycatchbranch(self, variable, block):
        """ not reachable since the try-catch construct is translated as one """
        raise Exception("not reachable")

    def trycatchlngbranch(self, variable, block):
        """ not reachable since the try-catch construct is translated as one """
        raise Exception("not reachable")

    def trycatchusrbranch(self, variable, block):
        """ not reachable since the try-catch construct is translated as one """
        raise Exception("not reachable")

    def variable(self, id: str, standalone: bool):
        """ Translates a variable

        By default this is translated to a python variable (ast.Name),
        but there are two special cases:

        1.  If the variable is called "this" we need to translate it to the corresponding keyword which is "self".

        2.  If the transpiler is in a class and the variable is a class attribute
            the variable needs to be prefixed with "self." to enable the attribute access in pythons

        """

        if id == "this":
            return ast.Name(id="self")

        # prefix variable with "v_" if the id is a python keyword
        py_id = escape_id(id)

        if standalone == True and self.state.is_class_variable(py_id) and py_id not in self.state.variables:
            return ast.Attribute(value=ast.Name(id="self"), attr=py_id)

        if py_id in self.state.built_ins:
            return self.setlx_access(py_id)

        return ast.Name(id=py_id)

    def variableignore(self):
        """ Translates a ignored SetlX variable to a "_" python variable """
        return ast.Name(id="_")

    def setlx_access(self, name):
        """ Generates a expression that accesses a variable from the setlx package """
        self.state.imports.add("setlx")
        return ast.Attribute(value=ast.Name(id="setlx"), attr=name)

    def setlx_function(self, name: str, args: list) -> ast.Call:
        """ Generates a expression that calls a given function from the setlx package """
        self.state.imports.add("setlx")
        return ast.Call(func=self.setlx_access(name), args=args, keywords=[])

    def unpack_error(self, target) -> ast.Assign:
        """ Generates a statement that unpacks the error object from a Exception

        In SetlX every object can be thrown as a exception.
        In Python all Exception musst be subclasses of Exception.
        This statement unpacks the error that is inside a exception.
        e.g.::

            e = setlx.unpack_error(e)

        """
        self.state.imports.add("setlx")
        unpack_call = self.setlx_function("unpack_error", [ast.Name(id='e')])
        self.state.variables.append(target.id)
        return ast.Assign(targets=[target], value=unpack_call)

    def to_method(self, id: str, static: bool) -> ast.Assign:
        """ Generates a statement that converts a static class function to a none static one

        In SetlX static class functions can be called as instance methods.
        To enable this in python, the static functions are overriden with
        a none static one in the init function.
        This statement does the overriding
        e.g. self.function = setlx.to_method(self,ClassName.function)
        """
        args = [ast.Name(id="self")]
        if static:
            args.append(ast.Attribute(
                value=ast.Name(
                    id=self.state.class_context.id), attr=id)
            )
            args.append(bool_true())
        else:
            args.append(ast.Name(id=id))

        return ast.Assign(
            targets=[ast.Attribute(
                value=ast.Name(id='self'),
                attr=id
            )],
            value=self.setlx_function("to_method", args)
        )

    def copy_params(self, params: list) -> ast.Assign:
        """ Creates the ast structure for deep copying procedure parameters

        This structure is used to copy all parameters at the beginning of a function.
        e.g. For the parameters x,y,z the code looks like this::

            [x,y,z] = setlx.copy([x,y,z])

        Parameters
        ----------
        params : list
            A list of strings with the name of the parameters
        """
        elts = ast.List(elts=[ast.Name(p) for p in params])
        return ast.Assign(targets=[elts],
                          value=self.setlx_function("copy", [elts]))

class ParserErrorListener(ErrorListener):
    """ An ANTLR error listener that only raises syntax errors """

    def __init__(self):
        ErrorListener.__init__(self)

    def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e):
        """ Raises a SyntaxError with the line, collumn and message as information """
        raise SyntaxError("line " + str(line) + ":" + str(column) + " " + msg)

def parse_input(input: InputStream) -> SetlXgrammarParser:
    """ Parses a input stream and returns the generated parser Object

    Only syntax errors in the code are thrown (see ParserErrorListener)
    """
    lexer = SetlXgrammarLexer(input)
    stream = CommonTokenStream(lexer)
    parser = SetlXgrammarParser(stream)
    parser._listeners = [ParserErrorListener()]
    return parser

def parse_expr(string: str):
    """ Parses a Setlx-expression-string and returns the generated AST """
    input = InputStream(string)
    parser = parse_input(input)
    return parser.expr(False).ex

def call_function(name: str, params: list) -> ast.Call:
    """ Generates a ast for calling a function with parameters """
    return ast.Call(func=ast.Name(id=name), args=params, keywords=[])

def bool_true() -> ast.NameConstant:
    """ Returns the ast for "True" """
    return ast.NameConstant(value=True)

def bool_false() -> ast.NameConstant:
    """ Returns the ast for "False" """
    return ast.NameConstant(value=False)

def escape_id(id: str) -> str:
    """ Some ids in Setlx code might be Python keywords.
        To avoid syntax errors, these ids are prefixed with "v_"
    """
    return f"v_{id}" if id in keyword.kwlist+["setlx"] else id

def import_call(stlx_file: str) -> ast.ImportFrom:
    """ Generates a ast structure for importing the given setlx file

    The function takes a SetlX file import, strips the .stlx extension and replaces all "-"
    with underscores.
    e.g. "depth-first-search.stlx" -> depth_first_search

    Based on this generated module name a wildcard import statement is generated.
    e.g. from depth_first_search import *
    """
    if stlx_file[-5:len(stlx_file)] == ".stlx":
        file = stlx_file[0:-5]
    file = file.replace("-", "_")
    return ast.ImportFrom(module=file, names=[ast.alias(name='*', asname=None)], level=0)

class NotSupported(Exception):
    """ This exception is thrown when a lexical element is not supported """

    def __init__(self, msg=""):
        Exception.__init__(self, msg)


# == smart_plug_mini.py Author: Zuinige Rijder ==============================
"""
Simple Python3 script to get history kWh consumption of broadlink Smart Plug Mini
using the broadlink server and append later kWh measurements to a
.csv file format per device.

Also write the days, weeks, months and years summaries to separate csv files.

I have 2 times model Smart plug SP3S-EU and tested with those.
Probably this also works for other broadlink SP mini models.

Do not forget to configure smart_plug_mini.cfg
"""

import configparser
from dataclasses import dataclass, field
from datetime import datetime
from io import TextIOWrapper
from zoneinfo import ZoneInfo
import json
import os
from pathlib import Path
import socket
import sys
import time
import traceback
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from dateutil.relativedelta import relativedelta

def arg_has(string: str) -> bool:
    """arguments has string"""
    for i in range(1, len(sys.argv)):
        if sys.argv[i].lower() == string:
            return True
    return False

D = arg_has("debug")

def dbg(line: str) -> bool:
    """print line if debugging"""
    if D:
        print(f'{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}: {line}')
    return D  # just to make a lazy evaluation expression possible

@dataclass
class FileInfo:
    """helper dataclass to store file information"""

    postfix: str
    path: Path = field(init=False)
    file: TextIOWrapper = field(init=False)
    header: str = field(default="Date, Cumulative kWh, Delta +kWh")
    opened: bool = field(default=False)

CSV_GLOBAL = FileInfo(
    postfix="",
    header="Date, Cumulative kWh, Delta +kWh, Days +kWh, Weeks +kWh, Months +kWh, Years +kWh",  # noqa
)
CSV_DAYS = FileInfo(postfix=".days")
CSV_WEEKS = FileInfo(postfix=".weeks")
CSV_MONTHS = FileInfo(postfix=".months")
CSV_YEARS = FileInfo(postfix=".years")

FILES = [CSV_GLOBAL, CSV_DAYS, CSV_WEEKS, CSV_MONTHS, CSV_YEARS]

LICENSE_ID = "e9a19a0e0d099eb8288d878687b4883a"

parser = configparser.ConfigParser()
parser.read("smart_plug_mini.cfg")
sp3s_settings = dict(parser.items("smart_plug_mini"))

LOCAL_TIMEZONE = sp3s_settings["local_timezone"].strip()
SERVER = sp3s_settings["server"].strip()
DEVICE_NAMES = sp3s_settings["device_names"].strip()
DEVICE_MACS = sp3s_settings["device_macs"].strip()
REPORT_TYPES = sp3s_settings["report_types"].strip()
START_DATES = sp3s_settings["start_dates"].strip()
TIME_FILTER = sp3s_settings["time_filter"].strip()

ZONE_INFO_SERVER = ZoneInfo("Asia/Shanghai")
ZONE_INFO_LOCAL = ZoneInfo(LOCAL_TIMEZONE)

_ = D and dbg(
    f"SERVER: {SERVER}\nDEVICE_NAMES: [{DEVICE_NAMES}]\nDEVICE_MACS: [{DEVICE_MACS}]\nREPORT_TYPES: [{REPORT_TYPES}]\nSTART_DATES: {START_DATES}\nTIME_FILTER: {TIME_FILTER}"  # noqa
)

# these will be filled in a loop per device
DEVICE_NAME = ""
DEVICE_TYPE = ""
DEVICE_ID = ""
DATE_START = ""

def log(msg: str) -> None:
    """log a message prefixed with a date/time format yyyymmdd hh:mm:ss"""
    print(datetime.now().strftime("%Y%m%d %H:%M:%S") + ": " + msg)

def same_year(d_1: datetime, d_2: datetime) -> bool:
    """return if same year"""
    return d_1.year == d_2.year

def same_month(d_1: datetime, d_2: datetime) -> bool:
    """return if same month"""
    if d_1.month != d_2.month:
        return False
    return d_1.year == d_2.year

def same_week(d_1: datetime, d_2: datetime) -> bool:
    """return if same week"""
    if d_1.isocalendar().week != d_2.isocalendar().week:
        return False
    return d_1.year == d_2.year

def same_day(d_1: datetime, d_2: datetime) -> bool:
    """return if same day"""
    if d_1.day != d_2.day:
        return False
    if d_1.month != d_2.month:
        return False
    return d_1.year == d_2.year

# == post ====================================================================
def execute_request(url, data, headers) -> str:
    """execute request and handle errors"""
    if D:
        print(f"execute_request url={url}")
        print(f"execute_request post_data={data}")
        print(f"execute_request headers={headers}")
        print()

    retries = 2
    while retries > 0:
        retries -= 1
        request = Request(url, data=data, headers=headers)

        errorstring = ""
        try:
            with urlopen(request, timeout=30) as response:
                body = response.read()
                content = body.decode("utf-8")
                _ = D and dbg(f"execute_request content: {content}\n")
                return content
        except HTTPError as error:
            errorstring = str(error.status) + ": " + error.reason
        except URLError as error:
            errorstring = str(error.reason)
        except TimeoutError:
            errorstring = "Request timed out"
        except socket.timeout:
            errorstring = "Socket timed out"
        except Exception as ex:  # pylint: disable=broad-except
            errorstring = "urlopen exception: " + str(ex)
            traceback.print_exc()

        log("ERROR: " + url + " -> " + errorstring)
        time.sleep(60)  # retry after 1 minute
    return "ERROR"

def get_last_line(filename: Path) -> str:
    """get last line of filename"""
    last_line = ""
    if filename.is_file():
        with open(filename.name, "rb") as file:
            try:
                file.seek(-2, os.SEEK_END)
                while file.read(1) != b"\n":
                    file.seek(-2, os.SEEK_CUR)
            except OSError:
                file.seek(0)
            last_line = file.readline().decode().strip()
    print(f"last line {filename.name}: {last_line}")
    return last_line

def close_files() -> None:
    """close files"""
    for fileinfo in FILES:
        if fileinfo.opened:
            print(f"Closing {fileinfo.path.name}")
            fileinfo.file.close()
            fileinfo.opened = False

def write_line(fileinfo: FileInfo, line: str) -> None:
    """write line"""
    if not fileinfo.opened:
        write_header = False
        fileinfo.path = Path(f"{DEVICE_NAME}{fileinfo.postfix}.csv")
        if not fileinfo.path.is_file():
            fileinfo.path.touch()
            write_header = True
        print(f"Opening {fileinfo.path.name}")
        fileinfo.file = fileinfo.path.open("a", encoding="utf-8")
        fileinfo.opened = True
        if write_header:
            print(fileinfo.header)
            fileinfo.file.write(fileinfo.header)
            fileinfo.file.write("\n")

    print(f"{fileinfo.path.name}: {line}")
    fileinfo.file.write(line)
    fileinfo.file.write("\n")

def get_kwh_counters(date_start_str: str, date_end_str: str) -> dict:
    """get_kwh_counters"""
    post = f"{SERVER}/dataservice/v2/device/stats"
    headers = {
        "Content-Type": "text/plain; charset=UTF-8",
        "Connection": "close",
        "Expect": "100-continue",
    }
    post_data = f'{{"credentials":{{"licenseid":"{LICENSE_ID}"}},"report":"{DEVICE_TYPE}","device":[{{"did":"{DEVICE_ID}","start":"{date_start_str}","end":"{date_end_str}","params":["elec"],{TIME_FILTER}}}]}}'  # noqa
    content = execute_request(post, post_data.encode("utf-8"), headers)
    if content == "ERROR":
        log(f"month_stats error: {post_data}")
        return {}

    result = json.loads(content)
    if result["msg"] != "ok":
        log(f"get_kwh_counters error: {result}")
        return {}

    table = result["table"]
    _ = D and dbg(f"get_kwh_counters table = {table}")
    if len(table) == 0:
        return {}

    values = table[0]["values"]
    _ = D and dbg(f"get_kwh_counters values = {values}")
    return values

def get_last_info_from_csv(
    csv_filename: Path,
) -> tuple[datetime, float, float, float, float, float]:
    """get_last_info_from_csv"""
    date_start_server = datetime.strptime(DATE_START, "%Y-%m-%d %H:%M")
    date_start_server = date_start_server.replace(tzinfo=ZONE_INFO_LOCAL).astimezone(
        ZONE_INFO_SERVER
    )
    last_kwh = 0.0
    last_d = 0.0
    last_w = 0.0
    last_m = 0.0
    last_y = 0.0

    log(f"##### {csv_filename} ######################################")
    last_line = get_last_line(csv_filename)
    if last_line != "":
        splitted = last_line.split(",")
        last_date_str = splitted[0].strip()
        if last_date_str.startswith("20") and len(splitted) == 7:
            last_kwh = float(splitted[1].strip())
            last_d = float(splitted[3].strip())
            last_w = float(splitted[4].strip())
            last_m = float(splitted[5].strip())
            last_y = float(splitted[6].strip())
            last_date = datetime.strptime(last_date_str, "%Y-%m-%d %H:%M")
            last_date_server = last_date.replace(tzinfo=ZONE_INFO_LOCAL).astimezone(
                ZONE_INFO_SERVER
            )
            if last_date_server > date_start_server:
                date_start_server = last_date_server
                log(f"                  {last_date_str}")

    return date_start_server, last_kwh, last_d, last_w, last_m, last_y

def compute(
    fixed: tuple[float, float, datetime],  # delta_kwh, prev_kwh, prev_date
    cum_x: float,
    no_change: bool,
    fileinfo: FileInfo,
) -> float:
    """compute"""
    delta_kwh = fixed[0]
    new_delta_x = 0.0
    if no_change:
        new_delta_x = cum_x + delta_kwh
        _ = D and dbg(
            f"delta_kwh: {delta_kwh:.2f} delta_x: {cum_x:.2f} new_delta_x: {new_delta_x:.2f}"  # noqa
        )
    else:
        new_delta_x = delta_kwh
        _ = D and dbg(
            f"Change: delta_kwh: {delta_kwh:.2f} delta_x: {cum_x:.2f} new_delta_x: {new_delta_x:.2f}"  # noqa
        )
        # append the previous day/week/month/year value to the summary csv file
        prev_kwh = fixed[1]
        prev_date = fixed[2]
        line = f"{local_dt_str(prev_date)}, {prev_kwh:.2f}, {cum_x:.2f}"
        write_line(fileinfo, line)

    return new_delta_x

def local_dt_str(date: datetime) -> str:
    """local_dt_str"""
    date_str = date.astimezone(ZONE_INFO_LOCAL).strftime("%Y-%m-%d %H:%M")
    return date_str

def do_kwh_counters() -> None:
    """do_kwh_counters"""
    now_server = (
        datetime.now(tz=ZONE_INFO_LOCAL)
        .replace(second=0, microsecond=0)
        .astimezone(ZONE_INFO_SERVER)
    )
    (
        date_start_server,
        prev_kwh,
        cum_d,
        cum_w,
        cum_m,
        cum_y,
    ) = get_last_info_from_csv(Path(f"{DEVICE_NAME}.csv"))
    prev_date = date_start_server.astimezone(ZONE_INFO_LOCAL)
    date_start_server += relativedelta(hours=1)  # start with next hour
    log(f"date_start local: {date_start_server.astimezone(ZONE_INFO_LOCAL)}")
    log(f"now local       : {now_server.astimezone(ZONE_INFO_LOCAL)}")
    while date_start_server < now_server:
        date_end_server = date_start_server + relativedelta(months=1)
        log(
            f"{DEVICE_NAME}: from {date_start_server.astimezone(ZONE_INFO_LOCAL)} to {date_end_server.astimezone(ZONE_INFO_LOCAL)}"  # noqa
        )
        result = get_kwh_counters(
            date_start_server.strftime("%Y-%m-%d_%H:%M:%S"),
            date_end_server.strftime("%Y-%m-%d_%H:%M:%S"),
        )
        date_start_server = date_end_server
        date_start_server += relativedelta(hours=1)  # start with next hour
        if result is None:
            log("no results")
            continue

        for entry in result:
            _ = D and dbg(f"month_stats entry = {entry}")
            kwh = entry["elec"]
            occurtime = entry["occurtime"]
            # convert the occurtime to local timezone time
            date = (
                datetime.strptime(occurtime, "%Y-%m-%d_%H:%M:%S")
                .replace(tzinfo=ZoneInfo("Asia/Shanghai"))  # add timezone info
                .astimezone()  # to local timezone
            )
            delta_kwh = kwh - prev_kwh
            _ = D and dbg(
                f"kwh: {kwh:.2f}, prev: {prev_kwh:.2f}, delta: {delta_kwh:.2f}"
            )
            if delta_kwh < 0.0:  # damned, previous entry should be corrected?
                log(
                    f"WARNING: delta_elec < 0: current: {date} {kwh} prev: {prev_date} {prev_kwh}"  # noqa
                )

            _ = D and dbg(
                f"prev_date: {local_dt_str(prev_date)}\ncurr_date: {local_dt_str(date)}"  # noqa
            )

            fixed = (delta_kwh, prev_kwh, prev_date)
            cum_d = compute(fixed, cum_d, same_day(prev_date, date), CSV_DAYS)
            cum_w = compute(fixed, cum_w, same_week(prev_date, date), CSV_WEEKS)
            cum_m = compute(fixed, cum_m, same_month(prev_date, date), CSV_MONTHS)
            cum_y = compute(fixed, cum_y, same_year(prev_date, date), CSV_YEARS)

            line = f"{local_dt_str(date)}, {kwh:.2f}, {delta_kwh:.2f},  {cum_d:.2f},  {cum_w:.2f},  {cum_m:.2f},  {cum_y:.2f}"  # noqa
            write_line(CSV_GLOBAL, line)

            prev_kwh = kwh
            prev_date = date

    close_files()

def main() -> None:
    """main"""
    global DEVICE_NAME, DEVICE_TYPE, DEVICE_ID, DATE_START  # noqa pylint: disable=global-statement
    for index, mac in enumerate(DEVICE_MACS.split(",")):
        DEVICE_NAME = DEVICE_NAMES.split(",")[index].strip()
        DEVICE_TYPE = REPORT_TYPES.split(",")[index].strip()
        DEVICE_ID = mac.strip().replace(":", "").lower().rjust(32, "0")
        DATE_START = START_DATES.split(",")[index].strip()
        _ = D and dbg(
            f"main(): DEVICE_NAME: {DEVICE_NAME}, DEVICE_TYPE: {DEVICE_TYPE}, DEVICE_ID: {DEVICE_ID}, DATE_START: {DATE_START}"  # noqa
        )
        do_kwh_counters()

main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed May 26 19:33:02 2021

@author: Mrinmoy Bhattacharjee, PhD Scholar, EEE Dept., IIT Guwahati
"""

import librosa
from scipy.io import loadmat
import numpy as np
import matplotlib.pyplot as plt
import lib.misc as misc
import os
from dictances import bhattacharyya
from scipy.signal import medfilt

def mode_filtering(X, win_size):
    if win_size%2==0:
        win_size += 1
    X_smooth = X.copy()
    for i in range(int(win_size/2), len(X)-int(win_size/2)):
        win = X[i-int(win_size/2):i+int(win_size/2)]
        uniq_lab, uniq_counts = np.unique(win, return_counts=True)
        X_smooth[i] = uniq_lab[np.argmax(uniq_counts)]
    return X_smooth

def get_transition_track(X, win_size=3):
    # X[:,0] = savgol_filter(X[:,0], 3, polyorder=2)
    # X[:,1] = 1-X[:,0]
    half_win_size = int(win_size/2)
    bhattacharya_dist = np.zeros(np.shape(X)[0])
    for i in range(half_win_size,np.shape(X)[0]-half_win_size):
        left_hist = {
            'negative_mn':np.mean(X[i-half_win_size:i,0]),
            # 'negative_sd':np.std(X[i-half_win_size:i,0]),
            'positive_mn':np.mean(X[i-half_win_size:i,1]),
            # 'positive_sd':np.std(X[i-half_win_size:i,1]),
            }
        right_hist = {
            'negative_mn':np.mean(X[i+1:i+half_win_size+1,0]),
            # 'negative_sd':np.std(X[i+1:i+half_win_size+1,0]),
            'positive_mn':np.mean(X[i+1:i+half_win_size+1,1]),
            # 'positive_sd':np.std(X[i+1:i+half_win_size+1,1]),
            }
        bhattacharya_dist[i] = bhattacharyya(left_hist, right_hist)

    # bhattacharya_dist[np.squeeze(np.where(bhattacharya_dist==0))] = np.min(bhattacharya_dist)
    bhattacharya_dist[np.squeeze(np.where(bhattacharya_dist>(np.mean(bhattacharya_dist)+5*np.std(bhattacharya_dist))))] = np.min(bhattacharya_dist)
    # bhattacharya_dist = savgol_filter(bhattacharya_dist, win_size, polyorder=2)

    return bhattacharya_dist

def draw_figures(opDir, feature_type, fold, segmentation_track, fl, MU_MARKER, CNN_PtdLabels_mu, LSTM_PtdLabels_mu, CNN_Frame_Predictions, SP_MARKER, CNN_PtdLabels_sp, LSTM_PtdLabels_sp, smoothing_win_size):
    if segmentation_track:
        BD_win_size = smoothing_win_size
        Transition_Track_mu = get_transition_track(CNN_Frame_Predictions['Pred'], BD_win_size)
        Transition_Track_sp = get_transition_track(CNN_Frame_Predictions['Pred'], BD_win_size)

    # plt.figure()
    opDirFig = opDir + '/__segmentation_results/' + feature_type + '/figures/fold' + str(fold) + '/'
    if not os.path.exists(opDirFig):
        os.makedirs(opDirFig)

    if not segmentation_track:
        plt.title(fl.split('/')[0])

        plt.subplot(311)
        plt.plot(MU_MARKER, 'b-')
        plt.legend(['Music GroundTruth'])

        plt.subplot(312)
        plt.plot(CNN_PtdLabels_mu, 'g-')
        plt.legend(['CNN labels mu'])

        plt.subplot(313)
        plt.plot(LSTM_PtdLabels_mu, 'm-')
        plt.legend(['LSTM music labels'])

    else:
        plt.subplot(411)
        plt.plot(MU_MARKER, 'b-')
        plt.legend(['Music GroundTruth'])
        plt.title('Window size='+str(BD_win_size))

        plt.subplot(412)
        X = CNN_Frame_Predictions['Pred'][:,1]
        # X = savgol_filter(X, 5, polyorder=3)
        plt.plot(X, 'g-')
        plt.legend(['CNN predictions mu'])

        plt.subplot(413)
        plt.plot(Transition_Track_mu, 'm-')
        plt.legend(['Bhattacharya dist'])

        plt.subplot(414)
        plt.plot(np.diff(Transition_Track_mu), 'm-')
        plt.legend(['Differenced Bhattacharya dist'])

    # plt.show()
    plt.savefig(opDirFig+fl.split('/')[-1].split('.')[0]+'_mu.jpg', bbox_inches='tight')
    plt.cla()

    # plt.figure()

    if not segmentation_track:
        plt.title(fl.split('/')[0])

        plt.subplot(311)
        plt.plot(SP_MARKER, 'b-')
        plt.legend(['Speech GroundTruth'])

        plt.subplot(312)
        plt.plot(CNN_PtdLabels_sp, 'g-')
        plt.legend(['CNN labels sp'])

        plt.subplot(313)
        plt.plot(LSTM_PtdLabels_sp, 'm-')
        plt.legend(['LSTM speech labels'])

    else:
        plt.subplot(411)
        plt.plot(SP_MARKER, 'b-')
        plt.legend(['Speech GroundTruth'])
        plt.title('Window size='+str(BD_win_size))

        plt.subplot(412)
        X = CNN_Frame_Predictions['Pred'][:,1]
        # X = savgol_filter(X, 5, polyorder=3)
        plt.plot(X, 'g-')
        plt.legend(['CNN predictions sp'])

        plt.subplot(413)
        plt.plot(Transition_Track_sp, 'm-')
        plt.legend(['Bhattacharya dist'])

        plt.subplot(414)
        plt.plot(np.diff(Transition_Track_sp), 'm-')
        plt.legend(['Differenced Bhattacharya dist'])

    # plt.show()
    plt.savefig(opDirFig+fl.split('/')[-1].split('.')[0]+'_sp.jpg', bbox_inches='tight')
    plt.cla()

def get_segment_level_statistics(ref_marker, est_marker, class_name, fold, opFile, feature_type):
    # print(np.shape(ref_marker), np.shape(est_marker))
    TP = np.sum(np.multiply((np.array(ref_marker)==1), (np.array(est_marker)==1)))
    FP = np.sum(np.multiply((np.array(ref_marker)==0), (np.array(est_marker)==1)))
    FN = np.sum(np.multiply((np.array(ref_marker)==1), (np.array(est_marker)==0)))
    TN = np.sum(np.multiply((np.array(ref_marker)==0), (np.array(est_marker)==0)))
    print('segment_level_statistics:')
    print(TP, FN)
    print(FP, TN)

    Accuracy = (TP+TN)/(TP+FP+FN+TN)
    Precision_P = TP/(TP+FP)
    Recall_P = TP/(TP+FN)
    F1_P = 2*Precision_P*Recall_P/(Precision_P+Recall_P)

    Precision_N = TN/(TN+FN)
    Recall_N = TN/(TN+FP)
    F1_N = 2*Precision_N*Recall_N/(Precision_N+Recall_N)

    results = {}
    results['0'] = 'Class:'+class_name
    results['1'] = 'Fold:'+str(fold)
    results['2'] = 'Feature type:'+feature_type
    results['3'] = 'Accuracy:'+str(np.round(Accuracy, 4))
    # results['4'] = 'Precision '+class_name+':'+str(np.round(Precision_P, 4))
    # results['5'] = 'Recall '+class_name+':'+str(np.round(Recall_P, 4))
    # results['6'] = 'F-measure '+class_name+':'+str(np.round(F1_P, 4))
    # results['7'] = 'Precision non-'+class_name+':'+str(np.round(Precision_N, 4))
    # results['8'] = 'Recall non-'+class_name+':'+str(np.round(Recall_N, 4))
    # results['9'] = 'F-measure non-'+class_name+':'+str(np.round(F1_N, 4))
    results['4'] = 'Precision sp'+':'+str(np.round(Precision_P, 4))
    results['5'] = 'Recall sp'+':'+str(np.round(Recall_P, 4))
    results['6'] = 'F-measure sp'+':'+str(np.round(F1_P, 4))
    results['7'] = 'Precision mu'+':'+str(np.round(Precision_N, 4))
    results['8'] = 'Recall mu'+':'+str(np.round(Recall_N, 4))
    results['9'] = 'F-measure mu'+':'+str(np.round(F1_N, 4))
    misc.print_analysis(opFile, results)

    print('Segment level statistics %s: A=%.2f, P=%.4f, R=%.4f, F=%.4f\n' % (class_name, Accuracy, Precision_P, Recall_P, F1_P))
    print('Segment level statistics non-%s: A=%.2f, P=%.4f, R=%.4f, F=%.4f\n' % (class_name, Accuracy, Precision_N, Recall_N, F1_N))

def get_basic_statistics(twin_frms, ref_marker, est_marker):
    TP = 0 # True Positives
    FP = 0 # False Positives
    FN = 0 # False Negatives

    est_event_frms = np.squeeze(np.where(est_marker>0))
    ref_event_frms = np.squeeze(np.where(ref_marker>0))
    if np.size(est_event_frms)==1:
        est_event_frms = [est_event_frms]
    if np.size(ref_event_frms)==1:
        ref_event_frms = [ref_event_frms]

    if (np.size(est_event_frms)<1):
        FN = np.size(ref_event_frms)
        return TP, FP, FN

    if (np.size(ref_event_frms)<1):
        FP = np.size(est_event_frms)
        return TP, FP, FN

    # ref_event_frms_count = np.zeros(len(ref_event_frms))
    # frm_diff_min = []
    # if np.size(est_event_frms)>0:
    #     for i in range(len(est_event_frms)):
    #         TP_flag = 0
    #         frm_diff = []
    #         for j in range(len(ref_event_frms)):
    #             frm_diff.append(np.abs(est_event_frms[i]-ref_event_frms[j]))
    #             if  np.abs(est_event_frms[i]-ref_event_frms[j])<=twin_frms:
    #                 TP_flag = j+1
    #                 ref_event_frms_count[j] += 1
    #                 break
    #         frm_diff_min.append(np.min(frm_diff))
    #         if (TP_flag>0):
    #             if ref_event_frms_count[TP_flag-1]==1:
    #                 TP += 1
    #             else:
    #                 FP += 1
    #         else:
    #             FP += 1
    # FN = len(ref_event_frms)-TP # np.sum(ref_event_frms_count==0)

    if np.size(ref_event_frms)>0:
        for j in range(len(ref_event_frms)):
            FN_flag = True
            for i in range(len(est_event_frms)):
                if  np.abs(est_event_frms[i]-ref_event_frms[j])<=twin_frms:
                    FN_flag = False
                    break
            if FN_flag:
                FN += 1
            else:
                TP += 1
    FP = len(est_event_frms)-TP

    # print('Average frame differences: ', np.mean(frm_diff_min), np.std(frm_diff_min))

    return TP, FP, FN

def get_final_statistics(TPc, FPc, FNc, Nc, opFile, class_name, fold, feature_type, twin, segFunc):
    # print('get_final_statistics: ', class_name, twin, TPc, FPc, FNc, Nc)
    Pc = TPc/(TPc+FPc+1e-10) # Precision
    Rc = TPc/(TPc+FNc+1e-10) # Recall
    Fc = 2*Pc*Rc/(Pc+Rc+1e-10) # F_measure
    Dc = FNc/(Nc+1e-10) # Deletion rate
    Ic = FPc/(Nc+1e-10) # Insertion rate
    Ec = Dc+Ic # Error rate

    results = {}
    results['0'] = 'Class:'+class_name
    results['1'] = 'Fold:'+str(fold)
    results['2'] = 'Feature type:'+ feature_type
    results['3'] = 'Tolerance:'+str(twin)+' ms'
    results['4'] = 'SegFunc:'+segFunc
    results['5'] = 'True Positives:'+str(TPc)
    results['6'] = 'False Positives:'+str(FPc)
    results['7'] = 'False Negatives:'+str(FNc)
    results['8'] = 'Precision:'+str(np.round(Pc, 4))
    results['9'] = 'Recall:'+str(np.round(Rc, 4))
    results['10'] = 'F-measure:'+str(np.round(Fc, 4))
    results['11'] = 'Deletion Rate:'+str(np.round(Dc, 4))
    results['12'] = 'Insertion Rate:'+str(np.round(Ic, 4))
    results['13'] = 'Error Rate:'+str(np.round(Ec, 4))
    misc.print_analysis(opFile, results)

    return Pc, Rc, Fc, Dc, Ic, Ec

def compute_segmentation_performance(event_type, Statistics, PtdLabels_sp, PtdLabels_mu, ref_speech_marker, ref_music_marker, tol_windows, win_size=5):
    ref_speech_marker = np.squeeze(ref_speech_marker)
    ref_music_marker = np.squeeze(ref_music_marker)
    PtdLabels_sp = np.squeeze(PtdLabels_sp)
    PtdLabels_mu = np.squeeze(PtdLabels_mu)
    for twin in tol_windows:
        nFrames = len(ref_speech_marker)
        # print('nFrames = ', nFrames, twin)
        # print('ref_speech_marker: ', np.shape(ref_speech_marker), np.unique(ref_speech_marker))
        # print('ref_music_marker: ', np.shape(ref_music_marker), np.unique(ref_music_marker))
        # print('PtdLabels_sp: ', np.shape(PtdLabels_sp), np.unique(PtdLabels_sp))
        # print('PtdLabels_mu: ', np.shape(PtdLabels_mu), np.unique(PtdLabels_mu))

        # d = np.shape(ref_speech_marker)[0]-np.shape(PtdLabels_sp)[0]
        # ref_speech_marker = ref_speech_marker[d:np.shape(PtdLabels_sp)[0]]
        # print('ref_speech_marker modified: ', d, np.shape(ref_speech_marker), np.unique(ref_speech_marker))
        # d = np.shape(ref_music_marker)[0]-np.shape(PtdLabels_mu)[0]
        # ref_music_marker = ref_music_marker[d:np.shape(PtdLabels_mu)[0]]
        # print('ref_music_marker modified: ', d, np.shape(ref_music_marker), np.unique(ref_music_marker))

        other_idx_sp = np.squeeze(np.where(ref_speech_marker<0))
        ref_speech_marker[other_idx_sp] = 0
        # print('ref_speech_marker modified: ', np.shape(ref_speech_marker), np.unique(ref_speech_marker))
        PtdLabels_sp[other_idx_sp] = 0
        # print('PtdLabels_sp modified: ', np.shape(PtdLabels_sp), np.unique(PtdLabels_sp))

        other_idx_mu = np.squeeze(np.where(ref_music_marker<0))
        ref_music_marker[other_idx_mu] = 0
        # print('ref_music_marker modified: ', np.shape(ref_music_marker), np.unique(ref_music_marker))
        PtdLabels_mu[other_idx_mu] = 0
        # print('PtdLabels_mu modified: ', np.shape(PtdLabels_mu), np.unique(PtdLabels_mu))

        if event_type=='onoff':
            ref_speech_marker_diff = np.abs(np.diff(ref_speech_marker))
            ref_music_marker_diff = np.abs(np.diff(ref_music_marker))
            est_speech_marker_diff = np.abs(np.diff(PtdLabels_sp))
            est_music_marker_diff = np.abs(np.diff(PtdLabels_mu))
        elif event_type=='on':
            ref_speech_marker_diff = (np.diff(ref_speech_marker)>0).astype(int)
            ref_music_marker_diff = (np.diff(ref_music_marker)>0).astype(int)
            est_speech_marker_diff = (np.diff(PtdLabels_sp)>0).astype(int)
            est_music_marker_diff = (np.diff(PtdLabels_mu)>0).astype(int)

        # print('ref_speech_marker_diff: ', np.shape(ref_speech_marker_diff), np.unique(ref_speech_marker_diff))
        # print('ref_music_marker_diff: ', np.shape(ref_music_marker_diff), np.unique(ref_music_marker_diff))
        # print('est_speech_marker_diff: ', np.shape(est_speech_marker_diff), np.unique(est_speech_marker_diff))
        # print('est_music_marker_diff: ', np.shape(est_music_marker_diff), np.unique(est_music_marker_diff))

        twin_frms = int(np.ceil((twin/(nFrames*10+15))*nFrames))
        TP_sp, FP_sp, FN_sp = get_basic_statistics(twin_frms, ref_speech_marker_diff, est_speech_marker_diff)
        Statistics[twin]['TP_sp'] += TP_sp
        Statistics[twin]['FP_sp'] += FP_sp
        Statistics[twin]['FN_sp'] += FN_sp
        Statistics[twin]['N_sp'] += np.sum(ref_speech_marker_diff)
        # print('%d Basic stats sp(%d): N_ref=%d, N_est=%d, TP=%d, FP=%d, FN=%d' % (twin, len(ref_speech_marker_diff), np.sum(ref_speech_marker_diff), np.sum(est_speech_marker_diff), TP_sp, FP_sp, FN_sp), end='\t', flush=True)

        TP_mu, FP_mu, FN_mu = get_basic_statistics(twin_frms, ref_music_marker_diff, est_music_marker_diff)
        Statistics[twin]['TP_mu'] += TP_mu
        Statistics[twin]['FP_mu'] += FP_mu
        Statistics[twin]['FN_mu'] += FN_mu
        Statistics[twin]['N_mu'] += np.sum(ref_music_marker_diff)
        # print('%d Basic stats mu (%d): N_ref=%d, N_est=%d, TP=%d, FP=%d, FN=%d' % (twin, len(ref_music_marker_diff), np.sum(ref_music_marker_diff), np.sum(est_music_marker_diff), TP_mu, FP_mu, FN_mu))

        prec_sp = TP_sp/(TP_sp+FP_sp+1e-10)
        rec_sp = TP_sp/(TP_sp+FN_sp+1e-10)
        fscore_sp = 2*prec_sp*rec_sp/(prec_sp+rec_sp+1e-10)
        prec_mu = TP_mu/(TP_mu+FP_mu+1e-10)
        rec_mu = TP_mu/(TP_mu+FN_mu+1e-10)
        fscore_mu = 2*prec_mu*rec_mu/(prec_mu+rec_mu+1e-10)
        # print(twin, fl, '\tFscore: ', prec_sp, rec_sp, fscore_sp, prec_mu, rec_mu, fscore_mu)
        # print('Win=(%d,%d,%d)\t(P_sp:%.4f, R_sp:%.4f, F1_sp:%.4f)\t(P_mu:%.4f, R_mu:%.4f, F1_mu:%.4f): ' % (twin, twin_frms, nFrames, prec_sp, rec_sp, fscore_sp, prec_mu, rec_mu, fscore_mu))
    # print('Stats: ', Stats)

    return Statistics.copy()

def print_segmentation_results(tolerance_windows, smoothing_win_size, opDir, event_type, feature_type, fold, plot_fig=False, segmentation_track=False, **kwargs):
    if 'segFunc' in kwargs.keys():
        segFunc = kwargs['segFunc']
    else:
        segFunc = 'LSTM'

    segmentation_path = opDir + '/__segmentation_results/' + feature_type + '/fold' + str(fold) + '/'
    seg_files = librosa.util.find_files(segmentation_path, ext=['mat'])
    classification_path = opDir + '/__Frame_Predictions_CNN/' + feature_type + '/'
    prediction_files = librosa.util.find_files(classification_path, ext=['pkl'])
    # print('segmentation_path: ', segmentation_path)
    # print('seg_files: ', seg_files)

    Statistics = None
    Statistics = {twin:{'TP_sp':0, 'FP_sp':0, 'FN_sp':0, 'N_sp':0, 'TP_mu':0, 'FP_mu':0, 'FN_mu':0, 'N_mu':0} for twin in tolerance_windows}
    for fl in seg_files:
        matching_files = [fl.split('/')[-1].split('.')[0]+'_fold'+str(fold) in pfl.split('/')[-1].split('.')[0] for pfl in prediction_files]
        matching_idx = np.squeeze(np.where(matching_files))
        # print('matching_files: ', matching_files, matching_idx)
        pred_fl = prediction_files[matching_idx]
        # print('\n\n\npred_fl: ', pred_fl.split('/')[-1], fl.split('/')[-1])
        CNN_Frame_Predictions = misc.load_obj(classification_path, pred_fl.split('/')[-1].split('.')[0])

        # print('Filename: ', fl)
        Segmentation_Results = loadmat(fl)
        # print(Segmentation_Results.keys())
        MU_MARKER = Segmentation_Results['music_groundtruth']
        SP_MARKER = Segmentation_Results['speech_groundtruth']

        # MU_MARKER = mode_filtering(MU_MARKER, smoothing_win_size)
        # SP_MARKER = mode_filtering(SP_MARKER, smoothing_win_size)

        cnn_pred = CNN_Frame_Predictions['Pred']
        if smoothing_win_size>0:
            cnn_pred[:,0] = medfilt(cnn_pred[:,0], smoothing_win_size)
            cnn_pred[:,1] = 1-cnn_pred[:,0]
        CNN_PtdLabels_mu = np.argmin(cnn_pred, axis=1)
        CNN_PtdLabels_sp = np.argmax(cnn_pred, axis=1)
        # print('CNN_Frame_Predictions: ', np.shape(MU_MARKER), np.shape(SP_MARKER), np.shape(CNN_PtdLabels_mu), np.shape(CNN_PtdLabels_sp))
        print('CNN accuracy (mu): ', np.sum(np.array(MU_MARKER[:,0])==np.array(CNN_PtdLabels_mu))/len(MU_MARKER))
        print('CNN accuracy (sp): ', np.sum(np.array(SP_MARKER[:,0])==np.array(CNN_PtdLabels_sp))/len(SP_MARKER))

        if segFunc=='LSTM':
            lstm_pred = Segmentation_Results['LSTM_Predictions']
            if smoothing_win_size>0:
                lstm_pred[:,0] = medfilt(lstm_pred[:,0], smoothing_win_size)
                lstm_pred[:,1] = 1-lstm_pred[:,0]
            LSTM_PtdLabels_mu = np.argmin(lstm_pred, axis=1)
            LSTM_PtdLabels_sp = np.argmax(lstm_pred, axis=1)
            # print('LSTM_Segmentation_Results: ', np.shape(LSTM_PtdLabels_mu), np.shape(LSTM_PtdLabels_sp))
            # print('Labels: sp', np.unique(SP_MARKER), ' mu', np.unique(MU_MARKER))
            Statistics = compute_segmentation_performance(event_type, Statistics.copy(), LSTM_PtdLabels_sp, LSTM_PtdLabels_mu, SP_MARKER, MU_MARKER, tolerance_windows, win_size=smoothing_win_size)
            if plot_fig:
                draw_figures(opDir, feature_type, fold, segmentation_track, fl, MU_MARKER, CNN_PtdLabels_mu, LSTM_PtdLabels_mu, CNN_Frame_Predictions, SP_MARKER, CNN_PtdLabels_sp, LSTM_PtdLabels_sp, smoothing_win_size)

        elif segFunc=='HMM':
            HMM_PtdLabels_sp = Segmentation_Results['HMM_PtdLabels']
            if smoothing_win_size>0:
                HMM_PtdLabels_sp = mode_filtering(HMM_PtdLabels_sp, smoothing_win_size)

            # HMM_PtdLabels_sp = np.squeeze(np.roll(np.array(HMM_PtdLabels_sp, ndmin=2).T, -2*smoothing_win_size, axis=0))

            HMM_PtdLabels_mu = 1-HMM_PtdLabels_sp
            # print('HMM_Segmentation_Results: ', np.shape(HMM_PtdLabels_mu), np.shape(HMM_PtdLabels_sp))
            # print('Labels: sp', np.unique(SP_MARKER), ' mu', np.unique(MU_MARKER))

            print('HMM accuracy (mu): ', np.sum(np.array(MU_MARKER[:,0])==np.array(HMM_PtdLabels_mu))/len(MU_MARKER))
            print('HMM accuracy (sp): ', np.sum(np.array(SP_MARKER[:,0])==np.array(HMM_PtdLabels_sp))/len(SP_MARKER))

            Statistics = compute_segmentation_performance(event_type, Statistics.copy(), HMM_PtdLabels_sp, HMM_PtdLabels_mu, SP_MARKER, MU_MARKER, tolerance_windows, win_size=smoothing_win_size)
            if plot_fig:
                draw_figures(opDir, feature_type, fold, segmentation_track, fl, MU_MARKER, CNN_PtdLabels_mu, HMM_PtdLabels_mu, CNN_Frame_Predictions, SP_MARKER, CNN_PtdLabels_sp, HMM_PtdLabels_sp, smoothing_win_size)

        elif segFunc=='CNN':
            # print('CNN_Segmentation_Results: ', np.shape(CNN_PtdLabels_mu), np.shape(CNN_PtdLabels_sp))
            # print('Labels: sp', np.unique(SP_MARKER), ' mu', np.unique(MU_MARKER))

            CNN_PtdLabels_sp = np.squeeze(np.roll(np.array(CNN_PtdLabels_sp, ndmin=2).T, -2000, axis=0))
            CNN_PtdLabels_mu = 1-CNN_PtdLabels_sp

            Statistics = compute_segmentation_performance(event_type, Statistics.copy(), CNN_PtdLabels_sp, CNN_PtdLabels_mu, SP_MARKER, MU_MARKER, tolerance_windows, win_size=smoothing_win_size)
            if plot_fig:
                draw_figures(opDir, feature_type, fold, segmentation_track, fl, MU_MARKER, CNN_PtdLabels_mu, CNN_PtdLabels_mu, CNN_Frame_Predictions, SP_MARKER, CNN_PtdLabels_sp, CNN_PtdLabels_sp, smoothing_win_size)

        # print(Statistics)

    opFile = opDir + '/Event_level_results.csv'
    for twin in tolerance_windows:
        P_sp, R_sp, F_sp, D_sp, I_sp, E_sp = get_final_statistics(Statistics[twin]['TP_sp'], Statistics[twin]['FP_sp'], Statistics[twin]['FN_sp'], Statistics[twin]['N_sp'], opFile, 'speech', fold, feature_type, twin, segFunc)

    for twin in tolerance_windows:
        P_mu, R_mu, F_mu, D_mu, I_mu, E_mu = get_final_statistics(Statistics[twin]['TP_mu'], Statistics[twin]['FP_mu'], Statistics[twin]['FN_mu'], Statistics[twin]['N_mu'], opFile, 'music', fold, feature_type, twin, segFunc)

    for twin in tolerance_windows:
        P, R, F, D, I, E = get_final_statistics(
            Statistics[twin]['TP_sp']+Statistics[twin]['TP_mu'],
            Statistics[twin]['FP_sp']+Statistics[twin]['FP_mu'],
            Statistics[twin]['FN_sp']+Statistics[twin]['FN_mu'],
            Statistics[twin]['N_sp']+Statistics[twin]['N_mu'],
            opFile, 'overall', fold, feature_type, twin, segFunc)

    # SP_MARKER = []
    # PtdLabels_sp = []
    # for fl in seg_files:
    #     Segmentation_Results = None
    #     Segmentation_Results = loadmat(fl)
    #     ref_marker = Segmentation_Results['speech_groundtruth']
    #     # ref_marker = mode_filtering(ref_marker, smoothing_win_size)

    #     if segFunc=='LSTM':
    #         lstm_pred = Segmentation_Results['LSTM_Predictions']
    #         if smoothing_win_size>0:
    #             lstm_pred[:,0] = medfilt(lstm_pred[:,0], smoothing_win_size)
    #             lstm_pred[:,1] = 1-lstm_pred[:,0]
    #         est_marker = np.argmax(lstm_pred, axis=1)
    #     elif segFunc=='HMM':
    #         est_marker = Segmentation_Results['HMM_PtdLabels']
    #         if smoothing_win_size>0:
    #             est_marker = mode_filtering(est_marker, smoothing_win_size)
    #     elif segFunc=='CNN':
    #         cnn_pred = Segmentation_Results['CNN_Predictions']
    #         if smoothing_win_size>0:
    #             cnn_pred[:,0] = medfilt(cnn_pred[:,0], smoothing_win_size)
    #             cnn_pred[:,1] = 1-cnn_pred[:,0]
    #         est_marker = np.argmax(cnn_pred, axis=1)

    #     ref_marker = np.squeeze(ref_marker)
    #     est_marker = np.squeeze(est_marker)
    #     # print('ref_marker: ', np.shape(ref_marker), np.unique(ref_marker))
    #     # print('est_marker: ', np.shape(est_marker), np.unique(est_marker))
    #     non_other_idx = np.squeeze(np.where(ref_marker>=0))
    #     # print(np.shape(ref_marker), np.shape(est_marker), np.shape(non_other_idx))
    #     if np.size(non_other_idx)<=1:
    #         continue
    #     ref_marker = ref_marker[non_other_idx]
    #     est_marker = est_marker[non_other_idx]
    #     # print('ref_marker modified: ', np.shape(ref_marker), np.unique(ref_marker))
    #     # print('est_marker modified: ', np.shape(est_marker), np.unique(est_marker))

    #     acc = np.round(np.sum(np.array(est_marker)==np.array(ref_marker))/np.size(ref_marker),4)
    #     print('sp: ', fl.split('/')[-1], acc)

    #     SP_MARKER.extend(ref_marker)
    #     PtdLabels_sp.extend(est_marker)
    # opFile = opDir + '/Segment_level_results.csv'
    # get_segment_level_statistics(SP_MARKER, PtdLabels_sp, 'speech', fold, opFile, feature_type)

    # MU_MARKER = []
    # PtdLabels_mu = []
    # for fl in seg_files:
    #     Segmentation_Results = None
    #     Segmentation_Results = loadmat(fl)
    #     ref_marker = Segmentation_Results['music_groundtruth']
    #     # ref_marker = mode_filtering(ref_marker, smoothing_win_size)

    #     if segFunc=='LSTM':
    #         lstm_pred = Segmentation_Results['LSTM_Predictions']
    #         if smoothing_win_size>0:
    #             lstm_pred[:,0] = medfilt(lstm_pred[:,0], smoothing_win_size)
    #             lstm_pred[:,1] = 1-lstm_pred[:,0]
    #         est_marker = np.argmin(lstm_pred, axis=1)
    #     elif segFunc=='HMM':
    #         est_marker = 1-Segmentation_Results['HMM_PtdLabels']
    #         if smoothing_win_size>0:
    #             est_marker = mode_filtering(est_marker, smoothing_win_size)
    #     elif segFunc=='CNN':
    #         cnn_pred = Segmentation_Results['CNN_Predictions']
    #         if smoothing_win_size>0:
    #             cnn_pred[:,0] = medfilt(cnn_pred[:,0], smoothing_win_size)
    #             cnn_pred[:,1] = 1-cnn_pred[:,0]
    #         est_marker = np.argmin(cnn_pred, axis=1)

    #     ref_marker = np.squeeze(ref_marker)
    #     est_marker = np.squeeze(est_marker)
    #     # print('ref_marker: ', np.shape(ref_marker), np.unique(ref_marker))
    #     # print('est_marker: ', np.shape(est_marker), np.unique(est_marker))
    #     non_other_idx = np.squeeze(np.where(ref_marker>=0))
    #     if np.size(non_other_idx)<=1:
    #         continue
    #     ref_marker = ref_marker[non_other_idx]
    #     est_marker = est_marker[non_other_idx]
    #     # print('ref_marker modified: ', np.shape(ref_marker), np.unique(ref_marker))
    #     # print('est_marker modified: ', np.shape(est_marker), np.unique(est_marker))

    #     acc = np.round(np.sum(np.array(est_marker)==np.array(ref_marker))/np.size(ref_marker),4)
    #     print('mu: ', fl.split('/')[-1], acc)

    #     MU_MARKER.extend(ref_marker)
    #     PtdLabels_mu.extend(est_marker)
    # opFile = opDir + '/Segment_level_results.csv'
    # get_segment_level_statistics(MU_MARKER, PtdLabels_mu, 'music', fold, opFile, feature_type)