# SPDX-License-Identifier: GPL-2.0-or-later

import bpy
from bpy.props import (FloatProperty,
                        BoolProperty,
                        EnumProperty,
                        StringProperty,
                        IntProperty,
                        PointerProperty)

## update on prop change
def update_prop(self, context):
    print('Update trigger')

class PROJ_PGT_settings(bpy.types.PropertyGroup) :
    ## HIDDEN to hide the animatable dot thing
    stringprop : StringProperty(
        name="str prop",
        description="",
        default="")# update=None, get=None, set=None

    boolprop : BoolProperty(
        name="bool prop",
        description="",
        default=False, options={'HIDDEN'}) # options={'ANIMATABLE'},subtype='NONE', update=None, get=None, set=None

    IntProperty : IntProperty(
        name="int prop", description="", default=25, min=1, max=2**31-1, soft_min=1, soft_max=2**31-1, step=1, options={'HIDDEN'})#, subtype='PIXEL'

    ## property with update on change
    edit_lines_opacity : FloatProperty(
        name="edit lines Opacity", description="Change edit lines opacity for all grease pencils",
        default=0.5, min=0.0, max=1.0, step=3, precision=2, update=update_prop)

    ## enum (with Icon)
    keyframe_type : EnumProperty(
        name="Keyframe Filter", description="Only jump to defined keyframe type",
        default='ALL', options={'HIDDEN', 'SKIP_SAVE'},
        items=(
            ('ALL', 'All', '', 0), # 'KEYFRAME'
            ('KEYFRAME', 'Keyframe', '', 'KEYTYPE_KEYFRAME_VEC', 1),
            ('BREAKDOWN', 'Breakdown', '', 'KEYTYPE_BREAKDOWN_VEC', 2),
            ('MOVING_HOLD', 'Moving Hold', '', 'KEYTYPE_MOVING_HOLD_VEC', 3),
            ('EXTREME', 'Extreme', '', 'KEYTYPE_EXTREME_VEC', 4),
            ('JITTER', 'Jitter', '', 'KEYTYPE_JITTER_VEC', 5),
            ))

class MYID_PGT_full_options_settings(bpy.types.PropertyGroup) :
    import sys
    Bool_variable_name : bpy.props.BoolProperty(
        name="", description="", default=False, options={'ANIMATABLE'}, subtype='NONE', update=None, get=None, set=None)

    BoolVector_variable_name : bpy.props.BoolVectorProperty(
        name="", description="", default=(False, False, False), options={'ANIMATABLE'}, subtype='NONE', size=3, update=None, get=None, set=None)

    Collection_variable_name : bpy.props.CollectionProperty(
        type=None, name="", description="", options={'ANIMATABLE'})

    Enum_variable_name : bpy.props.EnumProperty(
        items=(
            ('FIRST', 'first element label', 'First element hover description', 0),#include icon name in fourth position
            ('SECOND', 'Second element label', 'second element hover description', 1),
            ),
        name="", description="", default=None, options={'ANIMATABLE'}, update=None, get=None, set=None)

    Float_variable_name : bpy.props.FloatProperty(
        name="", description="", default=0.0, min=sys.float_info.min, max=sys.float_info.max, soft_min=sys.float_info.min, soft_max=sys.float_info.max, step=3, precision=2, options={'ANIMATABLE'}, subtype='NONE', unit='NONE', update=None, get=None, set=None)

    FloatVector_variable_name : bpy.props.FloatVectorProperty(
        name="", description="", default=(0.0, 0.0, 0.0), min=sys.float_info.min, max=sys.float_info.max, soft_min=sys.float_info.min, soft_max=sys.float_info.max, step=3, precision=2, options={'ANIMATABLE'}, subtype='NONE', unit='NONE', size=3, update=None, get=None, set=None)

    Int_variable_name : bpy.props.IntProperty(
        name="", description="", default=0, min=-2**31, max=2**31-1, soft_min=-2**31, soft_max=2**31-1, step=1, options={'ANIMATABLE'}, subtype='NONE', update=None, get=None, set=None)

    IntVector_variable_name : bpy.props.IntVectorProperty(
        name="", description="", default=(0, 0, 0), min=-2**31, max=2**31-1, soft_min=-2**31, soft_max=2**31-1, step=1, options={'ANIMATABLE'}, subtype='NONE', size=3, update=None, get=None, set=None)

    String_variable_name : bpy.props.StringProperty(
        name="", description="", default="", maxlen=0, options={'ANIMATABLE'}, subtype='NONE', update=None, get=None, set=None)

    # place at the end of register()
    # bpy.types.Scene.mytoolprops = bpy.props.PointerProperty(type = GP_PG_ToolsSettings)
    # place at the end of unregister()
    # del bpy.types.Scene.mytoolprops

# classes=(
# PROJ_PGT_settings,
# )

def register():
    # for cls in classes:
    #     bpy.utils.register_class(cls)
    bpy.utils.register_class(PROJ_PGT_settings)
    bpy.types.Scene.pgroup_name = bpy.props.PointerProperty(type = PROJ_PGT_settings)

def unregister():
    # for cls in reversed(classes):
    #     bpy.utils.unregister_class(cls)
    bpy.utils.unregister_class(PROJ_PGT_settings)
    del bpy.types.Scene.pgroup_name


from arpeggio import Optional, ZeroOrMore, OneOrMore, EOF, StrMatch
from arpeggio import RegExMatch as _
from arpeggio import ParserPython
from arpeggio import PTNodeVisitor, visit_parse_tree
import sys
import copy

# From Arpeggio tutorial, prevents string from appearing in parse tree
class SuppressStr(StrMatch):
    suppress = True

# Parser Functions
def atomUnit():
    # Second option lists accepted atoms, can add more if necesary
    return [(SuppressStr('('), atomList, SuppressStr(')')),
            (Optional(['=','#']), _(r'Br|Cl|[BCFHINOPS]'), Optional(_(r'\d*')))]

def atomList():
    return ZeroOrMore(atomUnit)

def ring():
    return SuppressStr(':'), atomList, \
        SuppressStr(':'), Optional(['=','#'])

def fragment():
    return [ring, atomList]

def fragmentList():
    return fragment, ZeroOrMore(SuppressStr('-'),fragment), EOF

# Visitor class that walks parse tree
# Returns list of fragments,
# Each fragment is tuple of 2 lists,
# First list is starting branch,
# Second list is ring
# Any atomList in parenthesis is its own list
# Any atomUnit is tuple of Atom string and int tag
class AtomCloneVistor(PTNodeVisitor):
    def visit_fragmentList(self, node, children):
        return list(children)

    def visit_fragment(self, node, children):
        d = dict()
        for i in range(len(node)):
            if node[i].rule_name == "atomList":
                d[node[i].rule_name] = children[i]
            elif node[i].rule_name == "ring":
                d[node[i].rule_name], bond = children[i]
                d["ringBond"] = bond

        return d

    def visit_ring(self, node, children):
        if len(children) == 0 or children[0] in ['=','#']:
            return None
        bond = 1
        if len(children) == 2 and children[1] == '=':
            bond = 2
        elif len(children) == 2 and children[1] == '#':
            bond = 3

        return (children[0], bond)

    def visit_atomList(self, node, children):
        return list(children)

    def visit_atomUnit(self, node, children):
        # Parenthesized branch
        if node[0].rule_name == "atomList":
            return children[0]

        noBondChildren = children
        bond = 1
        tag = None
        if children[0] == '=':
            bond = 2
            noBondChildren = children[1:]
        elif children[0] == '#':
            bond = 3
            noBondChildren = children[1:]

        # Atom and tag
        if len(noBondChildren) == 2:
            d= dict()
            d["atom"] = noBondChildren[0]
            d["tag"] = int(noBondChildren[1])
            d["bond"] = bond
            return d
        # Atom without tag
        else:
            d= dict()
            d["atom"] = noBondChildren[0]
            d["bond"] = bond
            return d

def initParser():
    """
    Set parser global variable to arpeggio parser for our format
    """
    global parser
    parser = ParserPython(fragmentList)

def fragmentAdjacencyMatrix(fragment):
    """
    Takes tuple representing a fragment and returns adjacency matrix,
    label list, and tag dictionary.

    arguments:
    fragment - tuple representing fragment as returned from visitor class

    returns:
    adjacency matrix as list of lists, label matrix as list of strings,
    dictionary mapping tags to index
    """
    numAtoms = 0
    ringStart = None
    labels = []
    bonds = []
    tags = dict()
    fidxStack = [0]
    atomStack = [-1]
    struct = "atomList"
    if struct not in fragment:
        struct = "ring"
        ringStart = 0
    # Traverse fragment in DFS order with iterative stack
    while len(fidxStack) > 0:
        # Use struct and fidxStack to get current object
        obj = fragment[struct]
        for i in fidxStack:
            if i < len(obj):
                obj = obj[i]
            else:
                # When index at one level goes out of bounds,
                # return to previous level
                fidxStack.pop()
                # Check if finished with ring, add ring closing bond
                if len(fidxStack) == 0 and struct == "ring":
                    bonds.append((ringStart, atomStack[0], fragment["ringBond"]))
                atomStack.pop()
                if len(fidxStack) > 0:
                    fidxStack[-1] += 1
                obj = None
                break
        # Handle individual atom by adding tag, label, and implicit bond
        if isinstance(obj, dict):
            labels.append(obj["atom"])
            if "tag" in obj:
                # assert (obj["tag"] not in tags), "Duplicate tag in same fragment"
                tags[obj["tag"]] = numAtoms

            bonds.append((atomStack[-1],numAtoms,obj["bond"]))
            fidxStack[-1] += 1
            atomStack[-1] = numAtoms
            numAtoms += 1
        # Handle branch (atomList) by setting up parent atom for next bond
        elif isinstance(obj, list):
            fidxStack.append(0)
            # atomStack.append(numAtoms-1)
            atomStack.append(atomStack[-1])

    # Remove dummy starter bond
    bonds = bonds[1:]

    # Make adjacency matrix with all zeros
    mat = []
    for i in range(numAtoms):
        mat.append([0]*numAtoms)

    for (i,j, b) in bonds:
        mat[i][j] = b
        mat[j][i] = b

    return mat, labels, tags

def mergeFragments(a, b):
    """
    Merges two fragment adjacency matrices and labels using tags.

    arguments:
    a - Fragment tuple (matrix, label, tag)
    b - Second fragment tuple (matrix, label, tag)

    returns:
    Merged fragment tuple (matrix, label, tag)
    """
    matA = a[0]
    labA = a[1]
    tagA = a[2]

    matB = b[0]
    labB = b[1]
    tagB = b[2]

    # Compute final amount of atoms in mergeed fragment
    newNumAtom = len(labA) + len(labB)
    intersect = [k for k in tagB if k in tagA]
    newNumAtom -= len(intersect)

    newLab = copy.copy(labA)
    # Construct list mapping index in B to combined index (accounting for tags)
    idxB = [None] * len(labB)
    for k in intersect:
        assert labA[tagA[k]] == labB[tagB[k]], "Same tag on different atom types"
        idxB[tagB[k]] = tagA[k]

    currIdx = len(labA)
    for i in range(len(idxB)):
        if idxB[i] is None:
            idxB[i] = currIdx
            newLab.append(labB[i])
            currIdx += 1
    for k in tagB.keys():
        tagB[k] = idxB[tagB[k]]
    newTag = {**tagB, **tagA}

    newMat = []
    for i in range(newNumAtom):
        newMat.append([0]*newNumAtom)

    # Copy A matrix
    for i in range(len(matA)):
        for j in range(len(matA)):
            newMat[i][j] = matA[i][j]

    # Add bonds from B matrix
    for i in range(len(matB)):
        for j in range(len(matB)):
            newMat[idxB[i]][idxB[j]] = max(matB[i][j],
                    newMat[idxB[i]][idxB[j]])

    return (newMat, newLab, newTag)

def strToAdjacencyMatrix(s):
    """
    Parses string and returns adjacency matrix and label array

    arguments:
    s - string to be parsed

    returns:
    adjacency matrix as list of lists, label matrix as list of strings
    """
    initParser()
    tree = parser.parse(s)
    listForm = visit_parse_tree(tree, AtomCloneVistor())
    currFragment = fragmentAdjacencyMatrix(listForm[0])
    for nextList in listForm[1:]:
        nextFragment = fragmentAdjacencyMatrix(nextList)
        currFragment = mergeFragments(currFragment, nextFragment)

    return currFragment[0:2]

def printMatrix(mat):
    """
    Prints matrix (list of lists) in nice format
    (assumes all entries take same number of characters)

    arguments:
    mat - list of lists representing matrix
    """
    for row in mat:
        print(' '.join([str(x) for x in row]))

def printLabels(lab):
    """
    Prints label list in nice format

    arguments:
    mat - list of lists representing matrix
    """
    for i,c in enumerate(lab):
        print(str(i)+" "+c)

# initParser()
# test = ":C(C)C(C)C0C1S:-:C(CNC2)CCCO:-:C2C0C1NCN:"
# mat,lab = strToAdjacencyMatrix(test)
# print(test)
# printMatrix(mat)
# printLabels(lab)
#assert(len(sys.argv) == 2)

# initParser()

# with open(sys.argv[1]) as f:
#     for line in f:
#         cleanLine = line.strip()
#         print("String: ", cleanLine)
#         matrix, lab = strToAdjacencyMatrix(cleanLine)
#         printMatrix(matrix)
#         printLabels(lab)


import random
from typing import Any, Dict, Optional, Tuple, Union

import gym
import numpy as np

from pushworld.config import PUZZLE_EXTENSION
from pushworld.puzzle import (
    DEFAULT_BORDER_WIDTH,
    DEFAULT_PIXELS_PER_CELL,
    NUM_ACTIONS,
    PushWorldPuzzle,
)
from pushworld.utils.env_utils import get_max_puzzle_dimensions, render_observation_padded
from pushworld.utils.filesystem import iter_files_with_extension

class PushWorldEnv(gym.Env):
    """An OpenAI Gym environment for PushWorld puzzles.

    Rewards are calculated according to Appendix D of
    https://arxiv.org/pdf/1707.06203.pdf with one change: The negative reward per step
    is reduced to 0.01, since PushWorld puzzles tend to have longer solutions than
    Sokoban puzzles.

    Args:
        puzzle_path: The path of a PushWorld puzzle file or of a directory that
            contains puzzle files, possibly nested in subdirectories. All discovered
            puzzles are loaded, and the `reset` method randomly selects a new puzzle
            each time it is called.
        max_steps: If not None, the `step` method will return `done = True` after
            calling it `max_steps` times since the most recent call of `reset`.
        border_width: The pixel width of the border drawn to indicate object
            boundaries. Must be >= 1.
        pixels_per_cell: The pixel width and height of a discrete position in the
            environment. Must be >= 1 + 2 * border_width.
        standard_padding: If True, all puzzles are padded to the maximum width and
            height of the puzzles in the `pushworld.config.BENCHMARK_PUZZLES_PATH`
            directory. If False, puzzles are padded to the maximum dimensions of
            all puzzles found in the `puzzle_path`.
    """

    def __init__(
        self,
        puzzle_path: str,
        max_steps: Optional[int] = None,
        border_width: int = DEFAULT_BORDER_WIDTH,
        pixels_per_cell: int = DEFAULT_PIXELS_PER_CELL,
        standard_padding: bool = False,
    ) -> None:
        self._puzzles = []
        for puzzle_file_path in iter_files_with_extension(
            puzzle_path, PUZZLE_EXTENSION
        ):
            self._puzzles.append(PushWorldPuzzle(puzzle_file_path))

        if len(self._puzzles) == 0:
            raise ValueError(f"No PushWorld puzzles found in: {puzzle_path}")
        if border_width < 1:
            raise ValueError("border_width must be >= 1")
        if pixels_per_cell < 3:
            raise ValueError("pixels_per_cell must be >= 3")

        self._max_steps = max_steps
        self._pixels_per_cell = pixels_per_cell
        self._border_width = border_width

        widths, heights = zip(*[puzzle.dimensions for puzzle in self._puzzles])
        self._max_cell_width = max(widths)
        self._max_cell_height = max(heights)

        if standard_padding:
            standard_cell_height, standard_cell_width = get_max_puzzle_dimensions()

            if standard_cell_height < self._max_cell_height:
                raise ValueError(
                    "`standard_padding` is True, but the maximum puzzle height in "
                    "BENCHMARK_PUZZLES_PATH is less than the height of the puzzle(s) "
                    "in the given `puzzle_path`."
                )
            else:
                self._max_cell_height = standard_cell_height

            if standard_cell_width < self._max_cell_width:
                raise ValueError(
                    "`standard_padding` is True, but the maximum puzzle width in "
                    "BENCHMARK_PUZZLES_PATH is less than the width of the puzzle(s) "
                    "in the given `puzzle_path`."
                )
            else:
                self._max_cell_width = standard_cell_width

        # Use a fixed arbitrary seed for reproducibility of results and for
        # deterministic tests.
        self._random_generator = random.Random(123)

        self._current_puzzle = None
        self._current_state = None

        self._action_space = gym.spaces.Discrete(NUM_ACTIONS)

        self._observation_space = gym.spaces.Box(
            low=0.0,
            high=1.0,
            shape=render_observation_padded(
                self._puzzles[0], self._puzzles[0].initial_state, self._max_cell_height, self._max_cell_width, self._pixels_per_cell, self._border_width,
            ).shape,
            dtype=np.float32,
        )

    @property
    def action_space(self) -> gym.spaces.Space:
        """Implements `gym.Env.action_space`."""
        return self._action_space

    @property
    def observation_space(self) -> gym.spaces.Space:
        """Implements `gym.Env.observation_space`."""
        return self._observation_space

    @property
    def metadata(self) -> Dict[str, Any]:
        """Implements `gym.Env.metadata`."""
        return {"render_modes": ["rgb_array"]}

    @property
    def render_mode(self) -> str:
        """Implements `gym.Env.render_mode`. Always contains "rgb_array"."""
        return "rgb_array"

    @property
    def current_puzzle(self) -> PushWorldPuzzle or None:
        """The current puzzle, or `None` if `reset` has not yet been called."""
        return self._current_puzzle

    def reset(
        self,
        seed: Optional[int] = None,
        options: Optional[dict] = None,
    ) -> Tuple[np.ndarray, dict]:
        """Implements `gym.Env.reset`.

        This function randomly selects a puzzle from those provided to the constructor
        and resets the environment to the initial state of the puzzle.

        Args:
            seed: If not None, the random number generator in this environment is reset
                with this seed.
            options: Unused. Required by the `gym.Env.reset` interface.

        Returns:
            A tuple of (observation, info). The observation contains the initial
            observation of the environment after the reset, and it is formatted as an
            RGB image with shape (height, width, 3) with `float32` type and values
            ranging from [0, 1]. The info dictionary is unused.
        """
        if seed is not None:
            self._random_generator = random.Random(seed)

        self._current_puzzle = self._random_generator.choice(self._puzzles)
        self._current_state = self._current_puzzle.initial_state
        self._current_achieved_goals = self._current_puzzle.count_achieved_goals(
            self._current_state
        )
        self._steps = 0

        observation = render_observation_padded(
            self._current_puzzle, self._current_state, self._max_cell_height, self._max_cell_width, self._pixels_per_cell, self._border_width,
        )
        info = {"puzzle_state": self._current_state}

        return observation, info

    def step(self, action: int) -> Union[Tuple[np.ndarray, float, bool, dict], Tuple[np.ndarray, float, bool, bool, dict]]:
        """Implements `gym.Env.step`.

        The returned observation is an RGB image of the new state of the environment,
        formatted as a `float32` array with shape (height, width, 3) and values ranging
        from [0, 1].
        """
        if not self._action_space.contains(action):
            raise ValueError("The provided action is not in the action space.")

        if self._current_state is None:
            raise RuntimeError("reset() must be called before step() can be called.")

        self._steps += 1
        previous_state = self._current_state
        self._current_state = self._current_puzzle.get_next_state(
            self._current_state, action
        )
        observation = render_observation_padded(
            self._current_puzzle, self._current_state, self._max_cell_height, self._max_cell_width, self._pixels_per_cell, self._border_width,
        )

        terminated = self._current_puzzle.is_goal_state(self._current_state)

        if terminated:
            reward = 10.0
        else:
            previous_achieved_goals = self._current_puzzle.count_achieved_goals(
                previous_state
            )
            current_achieved_goals = self._current_puzzle.count_achieved_goals(
                self._current_state
            )
            reward = current_achieved_goals - previous_achieved_goals - 0.01

        truncated = False if self._max_steps is None else self._steps >= self._max_steps
        info = {"puzzle_state": self._current_state}

        return observation, reward, terminated, truncated, info

    def render(self, mode='rgb_array') -> np.ndarray:
        """Implements `gym.Env.render`.

        Returns:
            An RGB image of the current state of the environment, formatted as a
            `uint8` array with shape (height, width, 3).
        """
        assert mode == 'rgb_array', 'mode must be rgb_array.'
        return self._current_puzzle.render(
            self._current_state,
            border_width=self._border_width,
            pixels_per_cell=self._pixels_per_cell,
        )


import numpy as np
import math
from prettytable import PrettyTable

def r0(ob, s):
    return math.sqrt((ob.x - s.x)**2 + (ob.y - s.y)**2)

def operation(init_data, dict_1, stor, dict_2, tc_list, x0_list, y0_list, i):
    #заполнение второй таблицы
    dict_2['r0'] = np.array([r0(ob, stor) for ob in init_data])
    dict_2['vtr0'] = dict_1['vt'] * dict_2['r0']
    dict_2['xvt/r0'] = dict_1['xvt'] / dict_2['r0']
    dict_2['yvt/r0'] = dict_1['yvt'] / dict_2['r0']
    dict_2['vt/r0'] = dict_1['vt'] / dict_2['r0']

    tc = sum(dict_2['vtr0'])#расчет total costs
    tc_list.append(tc)
    sum_xvtr = sum(dict_2['xvt/r0'])
    sum_yvtr = sum(dict_2['yvt/r0'])
    sum_vtr = sum(dict_2['vt/r0'])

    print(f'{i} приближение')
    table_2 = PrettyTable()
    for key, values in dict_2.items():
        table_2.add_column(key, values)
    table_2.add_row(['Сумма', '', '-', tc, sum_xvtr, sum_yvtr, sum_vtr])
    print(table_2)
    print(f'TC = {tc}')
    stor.x = sum_xvtr / sum_vtr #уточнение координат склада
    stor.y = sum_yvtr / sum_vtr
    print(f'x0 = {stor.x}\ny0 = {stor.y}')
    if i > 0:
        x0_list.append(stor.x)
        y0_list.append(stor.y)


from pyftpdlib.filesystems import AbstractedFS, FilesystemError
from io import BytesIO
import django
import sys
import os

# 将项目路径添加到系统搜寻路径当中，查找方式为从当前脚本开始，找到要调用的django项目的路径
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# 设置项目的配置文件 不做修改的话就是 settings 文件
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "webserver.settings")
django.setup()  # 加载项目配置

from api.harbor import FtpHarborManager
from api.exceptions import HarborError

class HarborFileSystem(AbstractedFS):
    def __init__(self, *args, **kwargs):
        super(HarborFileSystem, self).__init__(*args, **kwargs)
        self.bucket_name = self.root
        self.root = '/'
        self.client = FtpHarborManager()

    def realpath(self, path):
        return path

    def isdir(self, path):
        try:
            return self.client.ftp_is_dir(self.bucket_name, path.lstrip('/'))
        except HarborError as error:
            return False

    def isfile(self, path):
        try:
            return self.client.ftp_is_file(self.bucket_name, path.lstrip('/'))
        except HarborError as error:
            return False

    def islink(self, fs_path):
        return False

    def chdir(self, path):
        if self.isdir(path):
            self._cwd = self.fs2ftp(path)
        else:
            raise FilesystemError('Not a dir.')

    def stat(self, path):
        pass

    def lstat(self, path):
        pass

    def listdir(self, path):
        dir_list = []
        try:
            files = []
            files_generator = self.client.ftp_list_dir_generator(self.bucket_name, path[1:], per_num=2000)
            for _ in range(10):
                try:
                    data = next(files_generator)
                except StopIteration as error:
                    break
                files.append(data)
            for fi in files:
                for file in fi:
                    if file.fod is True:
                        if file.upt:
                            dir_list.append((file.name, file.upt, file.si))
                        else:
                            dir_list.append((file.name, file.ult, file.si))
                    else:
                        dir_list.append((file.name + '/', file.ult, 0))
        except HarborError as error:
            raise FilesystemError(error.msg)
        return dir_list

    def format_list(self, basedir, listing, ignore_err=True):
        assert isinstance(basedir, str), basedir

        # path = self.fs2ftp(basedir)
        for tmp in listing:
            if isinstance(tmp, tuple):
                filename, mtimestr, size = tmp
                if filename.endswith('/'):
                    perm = "drwxrwxrwx"
                    mtimestr = mtimestr.strftime('%b %d %X')[:-3]
                    filename = filename[:-1]
                else:
                    mtimestr = mtimestr.strftime('%b %d %X')[:-3]
                    perm = "-r-xr-xr-x"
                line = "%s %3s %-8s %-8s %8s %12s %s\r\n" % (
                    perm, 1, 'root', 'root', size, mtimestr, filename)

                if self.cmd_channel is not None:
                    yield line.encode("utf8", self.cmd_channel.unicode_errors)
                else:
                    yield line.encode("utf8")
            else:
                perm = "-r-xr-xr-x"
                line = "%s %3s %-8s %-8s %8s %12s %s\r\n" % (
                    perm, 1, 'root', 'root', '', '', tmp)

                if self.cmd_channel is not None:
                    yield line.encode("utf8", self.cmd_channel.unicode_errors)
                else:
                    yield line.encode("utf8")

    def format_mlsx(self, basedir, listing, perms, facts, ignore_err=True):
        assert isinstance(basedir, str), basedir

        ftp_path = self.fs2ftp(basedir)
        if len(listing) == 1:
            if isinstance(listing[0], tuple):
                if listing[0][0].endswith('/'):
                    _type = "dir"
                    perm = 'r'
                    filename = listing[0][0][:-1]
                    mtimestr = listing[0][1]
                    mtimestr = str(mtimestr).split('.')[0].replace('-', '').replace(':', '').replace(' ', '')
                    size = listing[0][2]
                    line = "type=%s;size=%d;perm=%s;modify=%s;unique=%s; %s\r\n" % (
                        _type, size, perm, mtimestr, '', filename)
                    if self.cmd_channel is not None:
                        yield line.encode("utf8", self.cmd_channel.unicode_errors)
                    else:
                        yield line.encode("utf8")
                else:
                    filename = listing[0][0]
                    mtimestr = listing[0][1]
                    mtimestr = str(mtimestr).split('.')[0].replace('-', '').replace(':', '').replace(' ', '')
                    size = listing[0][2]
                    perm = 'el'
                    _type = "file"
                    line = "type=%s;size=%d;perm=%s;modify=%s;unique=%s; %s\r\n" % (
                        _type, size, perm, mtimestr, '', filename)
                    if self.cmd_channel is not None:
                        yield line.encode("utf8", self.cmd_channel.unicode_errors)
                    else:
                        yield line.encode("utf8")
            else:
                ftp_path = os.path.join(ftp_path, listing[0])
                try:
                    data = self.client.ftp_get_obj(self.bucket_name, ftp_path[1:])
                    filename = data.name
                    mtimestr = data.upt
                    mtimestr = str(mtimestr).split('.')[0].replace('-', '').replace(':', '').replace(' ', '')
                    size = data.si
                    perm = 'el'
                    _type = "file"
                    line = "type=%s;size=%d;perm=%s;modify=%s;unique=%s; %s\r\n" % (
                        _type, size, perm, mtimestr, '', filename)
                    if self.cmd_channel is not None:
                        yield line.encode("utf8", self.cmd_channel.unicode_errors)
                    else:
                        yield line.encode("utf8")
                except HarborError as error:
                    raise FilesystemError(error.msg)
        else:
            for filename, mtimestr, size in listing:
                if filename.endswith('/'):
                    _type = "dir"
                    perm = 'r'
                    filename = filename[:-1]
                else:
                    perm = 'el'
                    _type = "file"
                mtimestr = str(mtimestr).split('.')[0].replace('-', '').replace(':', '').replace(' ', '')
                line = "type=%s;size=%d;perm=%s;modify=%s;unique=%s; %s\r\n" % (
                    _type, size, perm, mtimestr, '', filename)

                if self.cmd_channel is not None:
                    yield line.encode("utf8", self.cmd_channel.unicode_errors)
                else:
                    yield line.encode("utf8")

    def open(self, filename, mode):
        """Open a file returning its handler."""
        assert isinstance(filename, str), filename
        # print('function: open', filename, mode)
        ftp_path = self.fs2ftp(filename)
        mode = mode.lower()
        return FileHandler(self.bucket_name, ftp_path, self.client, mode)

    def mkdir(self, path):
        ftp_path = self.fs2ftp(path)
        try:
            self.client.ftp_mkdir(self.bucket_name, ftp_path[1:])
        except (HarborError, Exception) as error:
            raise FilesystemError(str(error))

    def rename(self, src, dst):
        new_name = os.path.basename(dst)
        new_dir = os.path.dirname(dst)
        try:
            self.client.ftp_move_rename(self.bucket_name, src[1:], new_name, new_dir)
        except HarborError as error:
            raise FilesystemError(f'rename dir is not supported, {str(error)}')
        except Exception as error:
            raise FilesystemError(str(error))

    def lexists(self, path):
        ftp_path = self.fs2ftp(path)

        if self.isdir(ftp_path) or self.isfile(ftp_path):
            return True
        else:
            return False

    def rmdir(self, path):
        ftp_path = self.fs2ftp(path)
        try:
            self.client.ftp_rmdir(self.bucket_name, ftp_path[1:])
        except (HarborError, Exception) as error:
            raise FilesystemError(str(error))

    def remove(self, path):
        ftp_path = self.fs2ftp(path)
        try:
            self.client.ftp_delete_object(self.bucket_name, ftp_path[1:])
        except (HarborError, Exception) as error:
            raise FilesystemError(str(error.msg))

    def getsize(self, path):
        ftp_path = self.fs2ftp(path)
        try:
            return self.client.ftp_get_obj_size(self.bucket_name, ftp_path[1:])
        except (HarborError, Exception) as error:
            raise FilesystemError(str(error.msg))

class FileHandler(object):
    def __init__(self, bucket_name, ftp_path, client, mode):
        self.bucket_name = bucket_name
        self.name = os.path.basename(ftp_path)
        self.ftp_path = ftp_path
        self.client = client
        self.closed = False
        self.file = BytesIO()
        self.offset = 0             # file pointer position
        self.is_breakpoint = False  # 标记是否断点续传
        self.write_generator = None
        self.read_generator = None
        self.mode = mode

    def ensure_init_write_generator(self, is_break_point=None):
        """
        确保已初始化 写生成器
        """
        if self.write_generator:
            return

        is_break_point = self.is_breakpoint if is_break_point is None else is_break_point
        try:
            self.write_generator = self.client.ftp_get_write_generator(
                self.bucket_name, self.ftp_path[1:], is_break_point)
            next(self.write_generator)
        except HarborError as error:
            raise FilesystemError(error.msg)

    def ensure_init_read_generator(self):
        if self.read_generator:
            return

        try:
            self.read_generator, ob = self.client.ftp_get_obj_generator(
                self.bucket_name, self.ftp_path[1:], offset=self.offset, per_size=4 * 1024 ** 2)
        except HarborError as error:
            raise FilesystemError(error.msg)

    def write(self, data):
        self.ensure_init_write_generator()
        self.file.write(data)
        if self.file.tell() >= 1024 ** 2 * 64:
            self._sync_cache()

        return len(data)

    def read(self, size=None):
        self.ensure_init_read_generator()
        try:
            data = next(self.read_generator)
        except Exception as error:
            return b''

        return data

    def close(self):
        # 写模式时，确认init写生成器创建对象，防止ftp上传空文件时没有创建对象的问题
        if 'w' in self.mode:
            self.ensure_init_write_generator()

        self._sync_cache()
        self.file.close()
        self.closed = True

    def seek(self, offset):
        self._sync_cache()      # seek前，同步可能缓存的数据
        if self.offset == offset:
            return

        self.offset = offset
        if self.offset == 0:
            self.is_breakpoint = False
        else:
            self.is_breakpoint = True

        # 当前文件指针变offset了，读、写生成器都需要根据offset从新init
        self.write_generator = None
        self.read_generator = None

    def _sync_cache(self):
        """
        缓存的文件数据同步到存储桶
        """
        lenght = self.file.tell()
        if lenght == 0:
            return

        self.ensure_init_write_generator()

        try:
            self.write_generator.send((self.offset, self.file.getvalue()))
        except HarborError as error:
            raise FilesystemError(error.msg)

        self.offset += lenght
        self.file.truncate(0)       # 清缓存
        self.file.seek(0)           # 重置缓存指针

class DownLoader(object):
    def __init__(self, bucket_name, ftp_path, client):
        self.bucket_name = bucket_name
        self.name = os.path.basename(ftp_path)
        # self.name = '/'
        self.ftp_path = ftp_path
        self.client = client
        self.closed = False
        self.id = 0
        try:
            self.obj_generator, ob = self.client.ftp_get_obj_generator(
                self.bucket_name, self.ftp_path[1:], per_size=4 * 1024 ** 2)
        except HarborError as error:
            raise FilesystemError(error.msg)

    def read(self, size=None):
        try:
            data = next(self.obj_generator)
        except Exception as error:
            return b''

        return data

    def close(self):
        self.closed = True

if __name__ == '__main__':
    print('no info')
    f = open('harbor_auto.py')
    f.read()


from django.template import RequestContext
from django.shortcuts import render_to_response, redirect
from django.conf import settings
from django.http import HttpResponse
from django.contrib.auth.models import User
from django.contrib.auth.decorators import login_required

from datetime import datetime, date, time, timedelta

from users.models import UserProfile, DriverProfile, StaffProfile
from restaurants.models import Restaurant
from item.models import Item
from orders.models import Order, OrderItem
from cal.models import Entry

def view_kitchen(request):
	context = RequestContext(request)
	user = request.user

	able_to_access = False
	for p in StaffProfile.objects.all():
		if p.user == user:
			able_to_access = True

	if able_to_access:

		today = datetime.today().date()

		orders_to_display = []

		for entry in Entry.objects.filter(date=today).order_by('time'):
			for order in Order.objects.filter(entry=entry):
				orders_to_display.append(order)

		no_orders = False
		if len(orders_to_display) == 0:
			no_orders = True

		matrix = []
		items = Item.objects.all()
		entries = Entry.objects.filter(date=today).order_by('time')
		all_order_items = OrderItem.objects.all()

		for entry in entries:
			entry_item = []
			for i in items:
				item = i
				count = 0
				for oi in all_order_items:
					if oi.item == item:
						if (oi.order.entry == entry):
							count = count + oi.quantity
				to_add = [entry, item, count]
				entry_item.append(to_add)
			matrix.append(entry_item)

		tomorrow = today + timedelta(days=1)

		orders_for_tomorrow = []

		for entry in Entry.objects.filter(date=tomorrow).order_by('time'):
			for order in Order.objects.filter(entry=entry):
				orders_for_tomorrow.append(order)

		no_orders_tomorrow = False

		if len(orders_for_tomorrow) == 0:
			no_orders_tomorrow = True

		return render_to_response(
			"kitchen.html",
			{
			'matrix':matrix,
			'no_orders':no_orders,
			'no_orders_tomorrow':no_orders_tomorrow,
			'orders_to_display':orders_to_display,
			'orders_for_tomorrow':orders_for_tomorrow,
			'items':items,
			'entries':entries,
			},
			context)

	return HttpResponse("You are not credentialed to view this area.")


import logging
import multiprocessing
import warnings
from copy import copy
from typing import Dict, Optional, Tuple, Union, cast

import pandas as pd
import pyspark.sql.functions as F
from pandas import Series
from pyspark.ml import Transformer, PipelineModel
from pyspark.ml.util import MLWritable, MLReadable, MLWriter
from pyspark.ml.feature import VectorAssembler
from synapse.ml.lightgbm import LightGBMClassifier, LightGBMRegressor, LightGBMRegressionModel, LightGBMClassificationModel
from synapse.ml.onnx import ONNXModel
import lightgbm as lgb
from lightgbm import Booster, LGBMClassifier

from lightautoml.ml_algo.tuning.base import Distribution, SearchSpace
from lightautoml.pipelines.selection.base import ImportanceEstimator
from lightautoml.spark.dataset.base import SparkDataset
from lightautoml.spark.utils import SparkDataFrame
from lightautoml.spark.ml_algo.base import SparkTabularMLAlgo, SparkMLModel, AveragingTransformer
from lightautoml.spark.mlwriters import LightGBMModelWrapperMLReader, LightGBMModelWrapperMLWriter, ONNXModelWrapperMLReader, ONNXModelWrapperMLWriter
from lightautoml.spark.transformers.base import DropColumnsTransformer, PredictionColsTransformer, ProbabilityColsTransformer
from lightautoml.spark.validation.base import SparkBaseTrainValidIterator
from lightautoml.utils.timer import TaskTimer
from lightautoml.validation.base import TrainValidIterator

logger = logging.getLogger(__name__)

class LightGBMModelWrapper(Transformer, MLWritable, MLReadable):

    def __init__(self, model: Union[LightGBMRegressionModel, LightGBMClassificationModel] = None) -> None:
        super().__init__()
        self.model = model

    def write(self) -> MLWriter:
        return LightGBMModelWrapperMLWriter(self)

    @classmethod
    def read(cls):
        """Returns an MLReader instance for this class."""
        return LightGBMModelWrapperMLReader()

    def _transform(self, dataset: SparkDataset) -> SparkDataset:
        return self.model.transform(dataset)

class ONNXModelWrapper(Transformer, MLWritable, MLReadable):

    def __init__(self, model: ONNXModel = None) -> None:
        super().__init__()
        self.model = model

    def write(self) -> MLWriter:
        return ONNXModelWrapperMLWriter(self)

    @classmethod
    def read(cls):
        """Returns an MLReader instance for this class."""
        return ONNXModelWrapperMLReader()

    def _transform(self, dataset: SparkDataset) -> SparkDataset:
        return self.model.transform(dataset)

class SparkBoostLGBM(SparkTabularMLAlgo, ImportanceEstimator):

    _name: str = "LightGBM"

    _default_params = {
        "learningRate": 0.05,
        "numLeaves": 128,
        "featureFraction": 0.7,
        "baggingFraction": 0.7,
        "baggingFreq": 1,
        "maxDepth": -1,
        "minGainToSplit": 0.0,
        "maxBin": 255,
        "minDataInLeaf": 5,
        # e.g. num trees
        "numIterations": 3000,
        "earlyStoppingRound": 50,
        # for regression
        "alpha": 1.0,
        "lambdaL1": 0.0,
        "lambdaL2": 0.0,
    }

    # mapping between metric name defined via SparkTask
    # and metric names supported by LightGBM
    _metric2lgbm = {
        "binary": {
            "auc": "auc",
            "aupr": "areaUnderPR"
        },
        "reg": {
            "r2": "rmse",
            "mse": "mse",
            "mae": "mae",
        },
        "multiclass": {
            "crossentropy": "cross_entropy"
        }
    }

    def __init__(self,
                 cacher_key: str,
                 default_params: Optional[dict] = None,
                 freeze_defaults: bool = True,
                 timer: Optional[TaskTimer] = None,
                 optimization_search_space: Optional[dict] = {},
                 use_single_dataset_mode: bool = True,
                 max_validation_size: int = 10_000,
                 chunk_size: int = 4_000_000,
                 convert_to_onnx: bool = False,
                 mini_batch_size: int = 5000,
                 seed: int = 42):
        SparkTabularMLAlgo.__init__(self, cacher_key, default_params, freeze_defaults, timer, optimization_search_space)
        self._probability_col_name = "probability"
        self._prediction_col_name = "prediction"
        self._raw_prediction_col_name = "raw_prediction"
        self._assembler = None
        self._drop_cols_transformer = None
        self._use_single_dataset_mode = use_single_dataset_mode
        self._max_validation_size = max_validation_size
        self._seed = seed
        self._models_feature_impotances = []
        self._chunk_size = chunk_size
        self._convert_to_onnx = convert_to_onnx
        self._mini_batch_size = mini_batch_size

    def _infer_params(self) -> Tuple[dict, int]:
        """Infer all parameters in lightgbm format.

        Returns:
            Tuple (params, num_trees, early_stopping_rounds, verbose_eval, fobj, feval).
            About parameters: https://lightgbm.readthedocs.io/en/latest/_modules/lightgbm/engine.html

        """
        assert self.task is not None

        task = self.task.name

        params = copy(self.params)

        if "isUnbalance" in params:
            params["isUnbalance"] = True if params["isUnbalance"] == 1 else False

        verbose_eval = 1

        if task == "reg":
            params["objective"] = "regression"
            params["metric"] = self._metric2lgbm[task].get(self.task.metric_name, None)
        elif task == "binary":
            params["objective"] = "binary"
            params["metric"] = self._metric2lgbm[task].get(self.task.metric_name, None)
        elif task == "multiclass":
            params["objective"] = "multiclass"
            params["metric"] = "multiclass"
        else:
            raise ValueError(f"Unsupported task type: {task}")

        if task != "reg":
            if "alpha" in params:
                del params["alpha"]
            if "lambdaL1" in params:
                del params["lambdaL1"]
            if "lambdaL2" in params:
                del params["lambdaL2"]

        params = {**params}

        return params, verbose_eval

    def init_params_on_input(self, train_valid_iterator: TrainValidIterator) -> dict:
        self.task = train_valid_iterator.train.task

        sds = cast(SparkDataset, train_valid_iterator.train)
        rows_num = sds.data.count()
        task = train_valid_iterator.train.task.name

        suggested_params = copy(self.default_params)

        if self.freeze_defaults:
            # if user change defaults manually - keep it
            return suggested_params

        if task == "reg":
            suggested_params = {
                "learningRate": 0.05,
                "numLeaves": 32,
                "featureFraction": 0.9,
                "baggingFraction": 0.9,
            }

        if rows_num <= 10000:
            init_lr = 0.01
            ntrees = 3000
            es = 200

        elif rows_num <= 20000:
            init_lr = 0.02
            ntrees = 3000
            es = 200

        elif rows_num <= 100000:
            init_lr = 0.03
            ntrees = 2000
            es = 200
        elif rows_num <= 300000:
            init_lr = 0.04
            ntrees = 2000
            es = 100
        else:
            init_lr = 0.05
            ntrees = 2000
            es = 100

        if rows_num > 300000:
            suggested_params["numLeaves"] = 128 if task == "reg" else 244
        elif rows_num > 100000:
            suggested_params["numLeaves"] = 64 if task == "reg" else 128
        elif rows_num > 50000:
            suggested_params["numLeaves"] = 32 if task == "reg" else 64
            # params['reg_alpha'] = 1 if task == 'reg' else 0.5
        elif rows_num > 20000:
            suggested_params["numLeaves"] = 32 if task == "reg" else 32
            suggested_params["alpha"] = 0.5 if task == "reg" else 0.0
        elif rows_num > 10000:
            suggested_params["numLeaves"] = 32 if task == "reg" else 64
            suggested_params["alpha"] = 0.5 if task == "reg" else 0.2
        elif rows_num > 5000:
            suggested_params["numLeaves"] = 24 if task == "reg" else 32
            suggested_params["alpha"] = 0.5 if task == "reg" else 0.5
        else:
            suggested_params["numLeaves"] = 16 if task == "reg" else 16
            suggested_params["alpha"] = 1 if task == "reg" else 1

        suggested_params["learningRate"] = init_lr
        suggested_params["numIterations"] = ntrees
        suggested_params["earlyStoppingRound"] = es

        if task != "reg":
            if "alpha" in suggested_params:
                del suggested_params["alpha"]

        return suggested_params

    def _get_default_search_spaces(self, suggested_params: Dict, estimated_n_trials: int) -> Dict:
        """Train on train dataset and predict on holdout dataset.

        Args:
            fold_prediction_column: column name for predictions made for this fold
            full: Full dataset that include train and valid parts and a bool column that delimits records
            train: Train Dataset.
            valid: Validation Dataset.

        Returns:
            Target predictions for valid dataset.

        """
        assert self.task is not None

        optimization_search_space = dict()

        optimization_search_space["featureFraction"] = SearchSpace(
            Distribution.UNIFORM,
            low=0.5,
            high=1.0,
        )

        optimization_search_space["numLeaves"] = SearchSpace(
            Distribution.INTUNIFORM,
            low=4,
            high=255,
        )

        if self.task.name == "binary" or self.task.name == "multiclass":
            optimization_search_space["isUnbalance"] = SearchSpace(
                Distribution.DISCRETEUNIFORM,
                low=0,
                high=1,
                q=1
            )

        if estimated_n_trials > 30:
            optimization_search_space["baggingFraction"] = SearchSpace(
                Distribution.UNIFORM,
                low=0.5,
                high=1.0,
            )

            optimization_search_space["minSumHessianInLeaf"] = SearchSpace(
                Distribution.LOGUNIFORM,
                low=1e-3,
                high=10.0,
            )

        if estimated_n_trials > 100:
            if self.task.name == "reg":
                optimization_search_space["alpha"] = SearchSpace(
                    Distribution.LOGUNIFORM,
                    low=1e-8,
                    high=10.0,
                )

            optimization_search_space["lambdaL1"] = SearchSpace(
                Distribution.LOGUNIFORM,
                low=1e-8,
                high=10.0,
            )

        return optimization_search_space

    def predict_single_fold(self,
                            dataset: SparkDataset,
                            model: Union[LightGBMRegressor, LightGBMClassifier]) -> SparkDataFrame:

        temp_sdf = self._assembler.transform(dataset.data)

        pred = model.transform(temp_sdf)

        return pred

    def fit_predict_single_fold(self,
                                fold_prediction_column: str,
                                full: SparkDataset,
                                train: SparkDataset,
                                valid: SparkDataset) -> Tuple[SparkMLModel, SparkDataFrame, str]:
        assert self.validation_column in full.data.columns, 'Train should contain validation column'

        if self.task is None:
            self.task = full.task

        (
            params,
            verbose_eval
        ) = self._infer_params()

        logger.info(f"Input cols for the vector assembler: {full.features}")
        logger.info(f"Running lgb with the following params: {params}")

        if self._assembler is None:
            self._assembler = VectorAssembler(
                inputCols=self.input_features,
                outputCol=f"{self._name}_vassembler_features",
                handleInvalid="keep"
            )

        LGBMBooster = LightGBMRegressor if full.task.name == "reg" else LightGBMClassifier

        if full.task.name in ['binary', 'multiclass']:
            params['rawPredictionCol'] = self._raw_prediction_col_name
            params['probabilityCol'] = fold_prediction_column
            params['predictionCol'] = self._prediction_col_name
            params['isUnbalance'] = True
        else:
            params['predictionCol'] = fold_prediction_column

        master_addr = train.spark_session.conf.get('spark.master')
        if master_addr.startswith('local'):
            cores_str = master_addr[len("local["):-1]
            cores = int(cores_str) if cores_str != "*" else multiprocessing.cpu_count()
            params["numThreads"] = max(cores - 1, 1)
        else:
            params["numThreads"] = max(int(train.spark_session.conf.get("spark.executor.cores", "1")) - 1, 1)

        train_data = full.data
        valid_size = train_data.where(F.col(self.validation_column) == 1).count()
        max_val_size = self._max_validation_size
        if valid_size > max_val_size:
            warnings.warn(f"Maximum validation size for SparkBoostLGBM is exceeded: {valid_size} > {max_val_size}. "
                          f"Reducing validation size down to maximum.", category=RuntimeWarning)
            rest_cols = list(train_data.columns)
            rest_cols.remove(self.validation_column)

            replace_col = F.when(F.rand(self._seed) < max_val_size / valid_size, F.lit(True)).otherwise(F.lit(False))
            val_filter_cond = F.when(F.col(self.validation_column) == 1, replace_col).otherwise(F.lit(True))

            train_data = train_data.where(val_filter_cond)

        valid_data = valid.data

        lgbm = LGBMBooster(
            **params,
            featuresCol=self._assembler.getOutputCol(),
            labelCol=full.target_column,
            validationIndicatorCol=self.validation_column,
            verbosity=verbose_eval,
            useSingleDatasetMode=self._use_single_dataset_mode,
            isProvideTrainingMetric=True,
            chunkSize=self._chunk_size
        )

        logger.info(f"Use single dataset mode: {lgbm.getUseSingleDatasetMode()}. NumThreads: {lgbm.getNumThreads()}")

        if full.task.name == "reg":
            lgbm.setAlpha(0.5).setLambdaL1(0.0).setLambdaL2(0.0)

        ml_model = lgbm.fit(self._assembler.transform(train_data))

        val_pred = ml_model.transform(self._assembler.transform(valid_data))
        val_pred = DropColumnsTransformer(
            remove_cols=[],
            optional_remove_cols=[self._prediction_col_name, self._probability_col_name, self._raw_prediction_col_name]
        ).transform(val_pred)

        self._models_feature_impotances.append(ml_model.getFeatureImportances(importance_type='gain'))

        if self._convert_to_onnx:
            logger.info("Model convert is started")
            booster_model_str = ml_model.getLightGBMBooster().modelStr().get()
            booster = lgb.Booster(model_str=booster_model_str)
            model_payload_ml = self._convertModel(booster, len(self.input_features))

            onnx_ml = ONNXModel().setModelPayload(model_payload_ml)

            if full.task.name == "reg":
                onnx_ml = (
                    onnx_ml
                        .setDeviceType("CPU")
                        .setFeedDict({"input": f"{self._name}_vassembler_features"})
                        .setFetchDict({ml_model.getPredictionCol(): "variable"})
                        .setMiniBatchSize(self._mini_batch_size)
                )
            else:
                onnx_ml = (
                    onnx_ml
                        .setDeviceType("CPU")
                        .setFeedDict({"input": f"{self._name}_vassembler_features"})
                        .setFetchDict({ml_model.getProbabilityCol(): "probabilities", ml_model.getPredictionCol(): "label"})
                        .setMiniBatchSize(self._mini_batch_size)
                )

            ml_model = onnx_ml
            logger.info("Model convert is ended")

        return ml_model, val_pred, fold_prediction_column

    def fit(self, train_valid: SparkBaseTrainValidIterator):
        logger.info("Starting LGBM fit")
        self.fit_predict(train_valid)
        logger.info("Finished LGBM fit")

    def get_features_score(self) -> Series:
        imp = 0
        for model_feature_impotances in self._models_feature_impotances:
            imp = imp + pd.Series(model_feature_impotances)

        imp = imp / len(self._models_feature_impotances)

        result = Series(list(imp), index=self.features).sort_values(ascending=False)
        return result

    @staticmethod
    def _convertModel(lgbm_model: Booster, input_size: int) -> bytes:
        from onnxmltools.convert import convert_lightgbm
        from onnxconverter_common.data_types import FloatTensorType
        initial_types = [("input", FloatTensorType([-1, input_size]))]
        onnx_model = convert_lightgbm(lgbm_model, initial_types=initial_types, target_opset=9)
        return onnx_model.SerializeToString()

    def _build_transformer(self) -> Transformer:
        avr = self._build_averaging_transformer()
        if self._convert_to_onnx:
            wrapped_models = [ONNXModelWrapper(m) for m in self.models]
        else:
            wrapped_models = [LightGBMModelWrapper(m) for m in self.models]
        models = [el for m in wrapped_models for el in [m, DropColumnsTransformer(
                remove_cols=[],
                optional_remove_cols=[self._prediction_col_name,
                                    self._probability_col_name,
                                    self._raw_prediction_col_name]
            )
        ]]
        if self._convert_to_onnx:
            if self.task.name in ['binary', 'multiclass']:
                models.append(ProbabilityColsTransformer(probability_сols=self._models_prediction_columns, num_classes=self.n_classes))
            else:
                models.append(PredictionColsTransformer(prediction_сols=self._models_prediction_columns))
        averaging_model = PipelineModel(stages=[self._assembler] + models + [avr])
        return averaging_model

    def _build_averaging_transformer(self) -> Transformer:
        avr = AveragingTransformer(
            self.task.name,
            input_cols=self._models_prediction_columns,
            output_col=self.prediction_feature,
            remove_cols=[self._assembler.getOutputCol()] + self._models_prediction_columns,
            convert_to_array_first=not (self.task.name == "reg"),
            dim_num=self.n_classes
        )
        return avr

    def fit_predict(self, train_valid_iterator: SparkBaseTrainValidIterator) -> SparkDataset:
        """Fit and then predict accordig the strategy that uses train_valid_iterator.

        If item uses more then one time it will
        predict mean value of predictions.
        If the element is not used in training then
        the prediction will be ``numpy.nan`` for this item

        Args:
            train_valid_iterator: Classic cv-iterator.

        Returns:
            Dataset with predicted values.

        """
        logger.info("Starting LGBM fit")
        self.timer.start()

        self.input_roles = train_valid_iterator.input_roles

        res = super().fit_predict(train_valid_iterator)

        logger.info("Finished LGBM fit")
        return res


import torch
import torch.nn as nn

from fusionlab.layers.factories import ConvND, MaxPool
from fusionlab.utils import autopad

# ref: https://arxiv.org/abs/1409.4842
# Going Deeper with Convolutions
class ConvBlock(nn.Module):
    def __init__(self, cin, cout, kernel_size=3, spatial_dims=2, stride=1):
        super().__init__()
        self.conv = ConvND(spatial_dims, cin, cout, kernel_size, stride, padding=autopad(kernel_size))
        self.act = nn.ReLU(inplace=True)

    def forward(self, x):
        x = self.conv(x)
        x = self.act(x)
        return x

class InceptionBlock(nn.Module):
    def __init__(self, cin, dim0, dim1, dim2, dim3, spatial_dims=2):
        super().__init__()
        self.branch1 = ConvBlock(cin, dim0, 3, spatial_dims)
        self.branch3 = nn.Sequential(ConvBlock(cin, dim1[0], 1, spatial_dims),
                                     ConvBlock(dim1[0], dim1[1], 3, spatial_dims))
        self.branch5 = nn.Sequential(ConvBlock(cin, dim2[0], 1, spatial_dims),
                                     ConvBlock(dim2[0], dim2[1], 5, spatial_dims))
        self.pool = nn.Sequential(MaxPool(spatial_dims, 3, 1, autopad(3)),
                                  ConvBlock(cin, dim3, 3,spatial_dims))

    def forward(self, x):
        x0 = self.branch1(x)
        x1 = self.branch3(x)
        x2 = self.branch5(x)
        x3 = self.pool(x)
        x = torch.cat((x0, x1, x2, x3), 1)
        return x

class InceptionNetV1(nn.Module):
    def __init__(self, cin=3, spatial_dims=2):
        super().__init__()
        self.stem = nn.Sequential(
            ConvBlock(cin, 64, 7, spatial_dims, stride=2),
            MaxPool(spatial_dims, 3, 2, padding=autopad(3)),
            ConvBlock(64, 192, 3, spatial_dims),
            MaxPool(spatial_dims, 3, 2, padding=autopad(3)),
        )
        self.incept3a = InceptionBlock(192, 64, (96, 128), (16, 32), 32, spatial_dims)
        self.incept3b = InceptionBlock(256, 128, (128, 192), (32, 96), 64, spatial_dims)
        self.pool3 = MaxPool(spatial_dims, 3, 2, padding=autopad(3))
        self.incept4a = InceptionBlock(480, 192, (96, 208), (16, 48), 64, spatial_dims)
        self.incept4b = InceptionBlock(512, 160, (112, 224), (24, 64), 64, spatial_dims)
        self.incept4c = InceptionBlock(512, 128, (128, 256), (24, 64), 64, spatial_dims)
        self.incept4d = InceptionBlock(512, 112, (144, 288), (32, 64), 64, spatial_dims)
        self.incept4e = InceptionBlock(528, 256, (160, 320), (32, 128), 128, spatial_dims)
        self.pool4 = MaxPool(spatial_dims, 3, 2, padding=autopad(3))
        self.incept5a = InceptionBlock(832, 256, (160, 320), (32, 128), 128, spatial_dims)
        self.incept5b = InceptionBlock(832, 384, (192, 384), (48, 128), 128, spatial_dims)

    def forward(self, x):
        x = self.stem(x)
        x = self.incept3a(x)
        x = self.incept3b(x)
        x = self.pool3(x)
        x = self.incept4a(x)
        x = self.incept4b(x)
        x = self.incept4c(x)
        x = self.incept4d(x)
        x = self.incept4e(x)
        x = self.pool4(x)
        x = self.incept5a(x)
        x = self.incept5b(x)
        return x

if __name__ == "__main__":
    inputs = torch.normal(0, 1, (1, 3, 224, 224))
    outputs = InceptionBlock(3, 64, (96, 128), (16, 32), 32)(inputs)
    print(outputs.shape)
    assert list(outputs.shape) == [1, 256, 224, 224]

    outputs = InceptionNetV1()(inputs)
    print(outputs.shape)
    assert list(outputs.shape) == [1, 1024, 7, 7]

    inputs = torch.normal(0, 1, (1, 3, 224))
    outputs = InceptionNetV1(spatial_dims=1)(inputs)
    print(outputs.shape)
    assert list(outputs.shape) == [1, 1024, 7]


import json
from typing import Any, Optional

import requests
from requests.adapters import HTTPAdapter

from spark_pipeline_framework.utilities.slack.base_slack_client import BaseSlackClient

class SlackClient(BaseSlackClient):
    """
    Implements a basic Slack client in http to post messages to a channel
    Inspired by https://keestalkstech.com/2019/10/simple-python-code-to-send-message-to-slack-channel-without-packages/
    """

    def __init__(self, slack_token: str, channel: str, bot_user_name: str) -> None:
        """
        Client for sending messages to Slack

        :param slack_token: token to auth with Slack
        :param channel: channel to post message into
        :param bot_user_name: user name to use when posting messages
        """
        self.slack_token: str = slack_token
        self.slack_channel: str = channel
        self.slack_icon_url: str = "https://www.flaticon.com/free-icon/freepik_23317"
        self.slack_user_name: str = bot_user_name
        self.slack_thread: Optional[str] = None

    def post_message_to_slack(
        self, text: str, blocks: Any = None, use_conversation_threads: bool = True
    ) -> Any:
        """
        Posts a message to Slack
        :param text: message to post.  Can be markdown.
        :param blocks: (Optional) blocks to post
        :param use_conversation_threads: whether to send messages as reply to first message
        :return: response from Slack
        """
        if not text:
            return

        headers = {
            "Content-type": "application/json; charset=utf-8",
            "Authorization": f"Bearer {self.slack_token}",
        }
        adapter = HTTPAdapter()
        http = requests.Session()
        http.mount("https://", adapter)

        post = http.post(
            "https://slack.com/api/chat.postMessage",
            data=json.dumps(
                {
                    "channel": self.slack_channel,
                    "text": text,
                    "icon_url": self.slack_icon_url,
                    "username": self.slack_user_name,
                    "blocks": json.dumps(blocks) if blocks else None,
                    "thread_ts": self.slack_thread,
                }
            ),
            headers=headers,
        )
        response_json = post.json()
        if not self.slack_thread and use_conversation_threads:
            if "ok" in response_json and response_json["ok"] is True:
                # read thread id from response
                if "message" in response_json and "ts" in response_json["message"]:
                    self.slack_thread = response_json["message"]["ts"]
        return response_json


from daos.convo import ConvoDAO
from daos.user import UserDAO
from flask import Flask, jsonify, request

class convoHandler:
    def build_convo_dict(self, row):
        convo = {}
        convo['convo_id'] = row[0]
        convo['convo_users'] = row[1]
        convo['convo_userAmounts'] = row[2]
        convo['admin_id'] = row[3]
        convo['convo_name'] = row[4]

        return convo

    def build_post_dict(self, row):
        p = {}
        p['post_id'] = row[0]
        p['post_content'] = row[1]
        p['post_date'] = row[2]
        p['user_id'] = row[3]
        p['convo_id'] = row[4]
        p['hashtag_id'] = row[5]
        p['reaction_id'] = row[6]
        p['reply_id'] = row[7]
        p['reaction_amount'] = row[8]
        p['reply_amount'] = row[9]

    def build_convo_attributes (convo_id, convo_users, convo_userAmounts, admin_id, convo_name):
        convo = {}
        convo['convo_id'] = convo_id
        convo['convo_users'] = convo_users
        convo['convo_userAmounts'] = convo_userAmounts
        convo['admin_id'] = admin_id
        convo['name'] = convo_name

    def getAllConvos(self):
        dao = ConvoDAO()
        convos = dao.getAllConvos()
        convomapped = []
        for c in convos:
            convomapped.append(self.build_convo_dict(c))

        return jsonify(Conversations=convomapped)

    def searchByArgs(self, args):
        dao = ConvoDAO()

        param1 = args.get('admin_id')
        param2 = args.get('convo_userAmounts')
        param3 = args.get('convo_name')

        if param1:
            result = dao.searchByAdmin(param1)
        elif param2:
            result = dao.getAllConvoAmount(param2)
        elif param3:
            result = dao.getAllConvoNames(param3)
        else:
            return jsonify(Error="NOT FOUND"), 404
        mapped_result = []
        for c in result:
            mapped_result.append(self.build_convo_dict(c))
        return jsonify(Conversations=mapped_result)

    def getConvoById(self, convo_id):
        dao = ConvoDAO()
        rconvo = dao.getConvoById(convo_id)
        if rconvo == None:
            return jsonify(Error="NOT FOUND"), 404
        else:
            convomap = self.build_convo_dict(rconvo)

        return jsonify(Conversation=convomap)

    def updateConvo(self, convo_id, form):
        return jsonify(UpdateConvo="OK"), 200

    def deleteConvo(self, convo_id):
        return jsonify(DeleteConvo="OK"), 200

    def postConvo(self, convo_id):
        return jsonify(CreateConvo="OK"), 201

    def getAllConvoUsers(self, convo_id):
        daoc = ConvoDAO()
        daou = UserDAO()
        usr = daou.getAllUsers()
        result = []
        rconvo = daoc.getConvoById(convo_id)

        if rconvo == None:
            return jsonify(Error="NOT FOUND"), 404
        else:
            for u in usr:
                if rconvo.__contains__(u):
                    result.append(u)
        return jsonify(Users=result)

    def replyPost(self, convo_id, post_id, reply_content):
        return jsonify(ReplyPost="OK"), 200


import pandas as pd

from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import cross_val_score
from hyperopt import fmin, tpe, Trials, space_eval

from helpers.model_helpers import save_pipeline, save_cv_scores
from data.db import log_model_metadata

def train_model(x_train, y_train, get_pipeline_function, model_uid, model, param_space, iterations, cv_strategy,
                cv_scoring, static_param_space, db_schema_name=None, db_conn=None, log_to_db=False):
    """
    Trains a machine learning model, optimizes the hyperparameters, and saves the serialized model into the
    MODELS_DIRECTORY.
    :param x_train: x_train dataframe
    :param y_train: y_train series
    :param get_pipeline_function: callable that takes model to produce a scikit-learn pipeline
    :param model_uid: model uid
    :param model: instantiated model
    :param param_space: the distribution of hyperparameters to search over
    :param iterations: number of trial to search for optimal hyperparameters
    :param cv_strategy: cross validation strategy
    :param cv_scoring: scoring method used for cross validation
    :param static_param_space: parameter search space valid for all models (e.g. feature engineering)
    :param db_schema_name: database schema to log metrics to
    :param db_conn: database connection
    :param log_to_db: Boolean of whether or not to log model info to a database
    :returns: scikit-learn pipeline
    """
    print(f'training {model_uid}...')
    pipeline = get_pipeline_function(model)
    if static_param_space:
        param_space.update(static_param_space)

    cv_scores_df = pd.DataFrame()

    def _model_objective(params):
        pipeline.set_params(**params)
        score = cross_val_score(pipeline, x_train, y_train, cv=cv_strategy, scoring=cv_scoring, n_jobs=-1)

        temp_cv_scores_df = pd.DataFrame(score)
        temp_cv_scores_df = temp_cv_scores_df.reset_index()
        temp_cv_scores_df['index'] = 'fold_' + temp_cv_scores_df['index'].astype(str)
        temp_cv_scores_df = temp_cv_scores_df.T
        temp_cv_scores_df = temp_cv_scores_df.add_prefix('fold_')
        temp_cv_scores_df = temp_cv_scores_df.iloc[1:]
        temp_cv_scores_df['mean'] = temp_cv_scores_df.mean(axis=1)
        temp_cv_scores_df['std'] = temp_cv_scores_df.std(axis=1)
        temp_params_df = pd.DataFrame(params, index=list(range(0, len(params) + 1)))
        temp_cv_scores_df = pd.concat([temp_params_df, temp_cv_scores_df], axis=1)
        temp_cv_scores_df = temp_cv_scores_df.dropna()
        nonlocal cv_scores_df
        cv_scores_df = cv_scores_df.append(temp_cv_scores_df)

        return 1 - score.mean()

    trials = Trials()
    best = fmin(_model_objective, param_space, algo=tpe.suggest, max_evals=iterations, trials=trials)
    best_params = space_eval(param_space, best)

    cv_scores_df = cv_scores_df.sort_values(by=['mean'], ascending=False)
    cv_scores_df = cv_scores_df.reset_index(drop=True)
    cv_scores_df = cv_scores_df.reset_index()
    cv_scores_df = cv_scores_df.rename(columns={'index': 'ranking'})
    save_cv_scores(cv_scores_df, model_uid, 'cv_scores')

    pipeline.set_params(**best_params)
    pipeline.fit(x_train, y_train)
    save_pipeline(pipeline, model_uid, 'model')
    if log_to_db:
        log_model_metadata(model_uid, db_schema_name, db_conn)
    return pipeline

def calibrate_fitted_model(pipeline, model_uid, x_validation, y_validation, calibration_method='sigmoid'):
    """
    Trains a CalibratedClassiferCV using the fitted model from the pipeline as the base_estimator.

    :param pipeline: Fitted pipeline
    :param model_uid: model uid
    :param x_validation: x validation set
    :param y_validation: y validation set
    :param calibration_method: the calibration method to use; either sigmoid or isotonic; default is sigmoid
    :returns: sklearn pipeline with the model step as a fitted CalibratedClassifierCV
    """
    model = pipeline.named_steps['model']
    pipeline.steps.pop(len(pipeline) - 1)
    x_validation = pipeline.transform(x_validation)
    calibrated_model = CalibratedClassifierCV(model, cv='prefit', method=calibration_method)
    calibrated_model.fit(x_validation, y_validation)
    pipeline.steps.append(['model', calibrated_model])
    save_pipeline(pipeline, model_uid, 'model')
    return pipeline


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2021/4/21 17:27
# @Author  : zyf
# @File    : fcn32s.py
# @Software: PyCharm
import torch
import torch.nn as nn
from torchvision import models
from vgg import vgg
from torchsummary import summary

"""
    构建全卷积网络FCN,使用的backbone是VGG16

"""

class FCN32s(nn.Module):
    def __init__(self, num_classes=21):
        super(FCN32s, self).__init__()
        self.num_classes = num_classes
        backbone = vgg()
        # 对原图进行卷积conv1、pool1后图像缩小为1 / 2  featuremap
        self.layer1 = backbone.layer1
        self.pool1 = backbone.pool1
        # 对图像进行第二次卷积conv2、pool2后图像缩小为1 / 4 featuremap
        self.layer2 = backbone.layer2
        self.pool2 = backbone.pool2
        # 对图像进行第三次卷积conv3、pool3后图像缩小为1 / 8 featuremap
        self.layer3 = backbone.layer3
        self.pool3 = backbone.pool3
        # 对图像进行第四次卷积conv4、pool4后图像缩小为1 / 16 featuremap
        self.layer4 = backbone.layer4
        self.pool4 = backbone.pool4
        # ；对图像进行第五次卷积conv5、pool5后图像缩小为1 / 32 featuremap
        self.layer5 = backbone.layer5
        # 这里可以直接 进行类别的输出卷积了
        self.pool5 = backbone.pool5

        """
        所有的层都是卷积层，故称为全卷积网络。
        CNN操作过程中的全连接编程卷积操作的conv6、conv7，需要使用1*1的卷积
        图像的featuremap的大小依然为原图的1 / 32,此时图像不再叫featuremap而是叫heatmap.
        """
        # self.fc6 = nn.Conv2d(in_channels=512, out_channels=4096, kernel_size=1)  # 512*7*7 -> 4096*7*7
        # self.drop6 = nn.Dropout()
        # self.fc7 = nn.Conv2d(in_channels=4096, out_channels=4096, kernel_size=1)  # 4096*7*7 -> 4096*7*7
        # self.drop7 = nn.Dropout()

        # 输出结果    # 4096*7*7 -> num_classes*7*7
        self.score = nn.Conv2d(in_channels=512, out_channels=self.num_classes, kernel_size=1)

        # 进行32倍上采样,使用双线性插值法，到达原始图像尺寸
        self.up_sample = nn.Upsample(scale_factor=32, mode='bilinear', align_corners=True)

    def forward(self, inputs):
        x = self.layer1(inputs)
        x = self.pool1(x)  # feature map 1/2     224*224 -> 128*128
        x = self.layer2(x)
        x = self.pool2(x)  # feature map 1/4     224*224 -> 56*56
        x = self.layer3(x)
        x = self.pool3(x)  # feature map 1/8     224*224 -> 28*28
        x = self.layer4(x)
        x = self.pool4(x)  # feature map 1/16    224*224 -> 14*14
        x = self.layer5(x)
        x = self.pool5(x)  # feature map 1/32    224*224 -> 7*7

        # x = self.fc6(x)  # 512*7*7 -> 4096*7*7
        # x = self.drop6(x)
        # x = self.fc7(x)  # 4096*7*7 -> 4096*7*7
        # x = self.drop7(x)
        x = self.score(x)  # 4096*7*7 -> num_classes*7*7
        x = self.up_sample(x)  # num_classes*7*7 ->  num_classes*(7x32)*(7x32)
        return x

if __name__ == '__main__':
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    net = FCN32s()
    net = net.to(device)
    print(net)
    data = torch.rand((2, 3, 224, 224))
    result = net(data.to(device))
    print(result.size())
    summary(net, (3, 224, 224))


"""
    This files handles the generic information about the agent of reference of the selected scenario
    and let choose and compute study agent information.
"""
import datetime as dt

from dash.dependencies import Input, Output, State
from dash.exceptions import PreventUpdate
import numpy as np
import plotly.graph_objects as go
import itertools

from grid2viz.src.manager import make_episode, make_network_agent_overview
from grid2viz.src.kpi import EpisodeTrace
from grid2viz.src.kpi import actions_model
from grid2viz.src.utils.graph_utils import (
    get_axis_relayout, relayout_callback, layout_def, layout_no_data, max_or_zero)
from grid2viz.src.kpi.maintenances import (hist_duration_maintenances)

from grid2viz.src.utils.common_graph import make_action_ts, make_rewards_ts

def register_callbacks_macro(app):
    @app.callback(
        [Output("rewards_timeserie", "figure"),
         Output("cumulated_rewards_timeserie", "figure"),],
        [Input('agent_study', 'data'),
         Input('relayoutStoreMacro', 'data')],
        [State("rewards_timeserie", "figure"),
         State("cumulated_rewards_timeserie", "figure"),
         State("agent_ref", "data"),
         State("scenario", "data"),
         State("agent_study", "modified_timestamp"),
         State("relayoutStoreMacro", "modified_timestamp")]
    )
    def load_reward_data_scatter(study_agent, relayout_data_store, rew_figure,
                                 cumrew_figure,
                                 ref_agent, scenario,
                                 agent_study_ts, relayoutStoreMacro_ts):
        """Compute and  create figure with instant and cumulated rewards of the study and ref agent"""
        rew_layout = rew_figure["layout"]
        cumrew_layout = cumrew_figure["layout"]
        if agent_study_ts is not None and relayoutStoreMacro_ts is not None:
            condition = (relayout_data_store is not None
                         and relayout_data_store["relayout_data"]
                         and relayoutStoreMacro_ts > agent_study_ts)
        else:
            condition = (relayout_data_store is not None
                         and relayout_data_store["relayout_data"])
        if condition:
            relayout_data = relayout_data_store["relayout_data"]
            rew_new_axis_layout = get_axis_relayout(rew_figure, relayout_data)
            cumrew_new_axis_layout = get_axis_relayout(cumrew_figure, relayout_data)
            if rew_new_axis_layout is not None or cumrew_new_axis_layout is not None:
                if rew_new_axis_layout is not None:
                    rew_layout.update(rew_new_axis_layout)
                if cumrew_new_axis_layout is not None:
                    cumrew_layout.update(cumrew_new_axis_layout)
                return rew_figure, cumrew_figure

        return make_rewards_ts(study_agent, ref_agent, scenario, rew_layout, cumrew_layout)

    @app.callback(
        Output("agent_study_pie_chart", "figure"),
        [Input('agent_study', 'data')],
        [State("agent_study_pie_chart", "figure"),
         State("scenario", "data")]
    )
    def update_action_repartition_pie(study_agent, figure, scenario):
        new_episode = make_episode(study_agent, scenario)
        figure['data'] = action_repartition_pie(new_episode)
        figure['layout'].update(
            actions_model.update_layout(
                figure["data"][0].values == (0, 0, 0),
                "No Actions for this Agent"))
        return figure

    def action_repartition_pie(agent):
        nb_actions = agent.action_data_table[
            ['action_line', 'action_subs', 'action_redisp']].sum()
        return [go.Pie(
            labels=["Actions on Lines", "Actions on Substations", "Redispatching Actions"],
            values=[nb_actions["action_line"], nb_actions["action_subs"], nb_actions["action_redisp"]]
        )]

    @app.callback(
        Output("network_actions", "figure"),
        [Input("agent_study", "data")],
        [State("scenario", "data")]
    )
    def update_network_graph(study_agent, scenario):
        episode = make_episode(study_agent, scenario)

        return make_network_agent_overview(episode)

    @app.callback(
        Output("timeseries_table", "data"),
        [Input("rewards_timeserie", "clickData"),
         Input("agent_log_selector", "value")],
        [State("timeseries_table", "data"),
         State("agent_study", "data")]
    )
    def add_timestamp(click_data, new_agent, data, agent_stored):
        if new_agent != agent_stored or click_data is None:
            if data is not None:
                return data
            else:
                return []
        time_stamp_str = click_data["points"][0]["x"]
        try:
            dt.datetime.strptime(time_stamp_str, '%Y-%m-%d %H:%M')
        except ValueError:
            time_stamp_str = time_stamp_str + " 00:00"
        new_data = {"Timestamps": time_stamp_str}
        if new_data not in data:
            data.append(new_data)
        return data

    @app.callback(
        Output("user_timestamps_store", "data"),
        [Input("timeseries_table", "data")]
    )
    def update_user_timestamps_store(timestamps):
        if timestamps is None:
            raise PreventUpdate
        return [dict(label=timestamp["Timestamps"], value=timestamp["Timestamps"])
                for timestamp in timestamps]

    @app.callback(
        Output("relayoutStoreMacro", "data"),
        [Input("usage_rate_graph_study", "relayoutData"),
         Input("action_timeserie", "relayoutData"),
         Input("overflow_graph_study", "relayoutData"),
         Input("rewards_timeserie", "relayoutData"),
         Input("cumulated_rewards_timeserie", "relayoutData")],
        [State("relayoutStoreMacro", "data")]
    )
    def relayout_store(*args):
        return relayout_callback(*args)

    @app.callback(
        [Output("indicator_score_output", "children"),
         Output("indicator_nb_overflow", "children"),
         Output("indicator_nb_action", "children")],
        [Input('agent_study', 'data'),
         Input("scenario", "data")]
    )
    def update_nbs(study_agent, scenario):
        new_episode = make_episode(study_agent, scenario)
        score = f'{get_score_agent(new_episode):,}'
        nb_overflow = f'{get_nb_overflow_agent(new_episode):,}'
        nb_action = f'{get_nb_action_agent(new_episode):,}'

        return score, nb_overflow, nb_action

    def get_score_agent(agent):
        score = agent.meta["cumulative_reward"]
        return round(score)

    def get_nb_overflow_agent(agent):
        return agent.total_overflow_ts["value"].sum()

    def get_nb_action_agent(agent):
        return int(agent.action_data_table[['action_line', 'action_subs']].sum(
            axis=1).sum())

    @app.callback(
        Output("agent_study", "data"),
        [Input('agent_log_selector', 'value')],
        [State("agent_study", "data"),
         State("scenario", "data")],
    )
    def update_study_agent(study_agent, stored_agent, scenario):
        if study_agent == stored_agent:
            raise PreventUpdate
        make_episode(study_agent, scenario)
        return study_agent

    @app.callback(
        [Output("overflow_graph_study", "figure"), Output(
            "usage_rate_graph_study", "figure")],
        [Input('agent_study', 'data'),
         Input('relayoutStoreMacro', 'data')],
        [State("overflow_graph_study", "figure"),
         State("usage_rate_graph_study", "figure"),
         State("scenario", "data"),
         State("agent_study", "modified_timestamp"),
         State("relayoutStoreMacro", "modified_timestamp")]
    )
    def update_agent_log_graph(study_agent, relayout_data_store,
                               figure_overflow, figure_usage, scenario,
                               agent_study_ts, relayoutStoreMacro_ts):

        if agent_study_ts is not None and relayoutStoreMacro_ts is not None:
            condition = (relayout_data_store is not None
                         and relayout_data_store["relayout_data"]
                         and relayoutStoreMacro_ts > agent_study_ts)
        else:
            condition = (relayout_data_store is not None
                         and relayout_data_store["relayout_data"])
        if condition:
            relayout_data = relayout_data_store["relayout_data"]
            layout_usage = figure_usage["layout"]
            new_axis_layout = get_axis_relayout(figure_usage, relayout_data)
            if new_axis_layout is not None:
                layout_usage.update(new_axis_layout)
                figure_overflow["layout"].update(new_axis_layout)
                return figure_overflow, figure_usage
        new_episode = make_episode(study_agent, scenario)
        figure_overflow["data"] = new_episode.total_overflow_trace.copy()
        maintenance_traces = EpisodeTrace.get_maintenance_trace(new_episode, ["total"])
        if len(maintenance_traces) != 0:
            maintenance_traces[0].update({"name": "Nb of maintenances"})
            figure_overflow["data"].append(maintenance_traces[0])

        hazard_traces = EpisodeTrace.get_hazard_trace(new_episode, ["total"]).copy()
        if len(hazard_traces) != 0:
            hazard_traces[0].update({"name": "Nb of hazards"})
            figure_overflow["data"].append(hazard_traces[0])

        attacks_trace = EpisodeTrace.get_attacks_trace(new_episode).copy()
        if len(attacks_trace) != 0:
            attacks_trace[0].update({"name": "Attacks"})
            figure_overflow["data"].append(attacks_trace[0])

        figure_usage["data"] = new_episode.usage_rate_trace

        return figure_overflow, figure_usage

    @app.callback(
        Output("action_timeserie", "figure"),
        [Input('agent_study', 'data'),
         Input('relayoutStoreMacro', 'data')],
        [State("action_timeserie", "figure"),
         State("agent_ref", "data"),
         State("scenario", "data"),
         State("agent_study", "modified_timestamp"),
         State("relayoutStoreMacro", "modified_timestamp")]
    )
    def update_actions_graph(study_agent, relayout_data_store, figure,
                             agent_ref, scenario,
                             agent_study_ts, relayoutStoreMacro_ts):
        if agent_study_ts is not None and relayoutStoreMacro_ts is not None:
            condition = (relayout_data_store is not None
                         and relayout_data_store["relayout_data"]
                         and relayoutStoreMacro_ts > agent_study_ts)
        else:
            condition = (relayout_data_store is not None
                         and relayout_data_store["relayout_data"])
        if condition:
            relayout_data = relayout_data_store["relayout_data"]
            layout = figure["layout"]
            new_axis_layout = get_axis_relayout(figure, relayout_data)
            if new_axis_layout is not None:
                layout.update(new_axis_layout)
                return figure

        return make_action_ts(study_agent, agent_ref, scenario, figure['layout'])

    action_table_name_converter = dict(
        timestep="Timestep",
        timestamp="Timestamp",
        timestep_reward="Reward",
        action_line="Action on line",
        action_subs="Action on sub",
        action_redisp="Action of redispatch",
        redisp_impact="Redispatch impact",
        line_name="Line name",
        sub_name="Sub name",
        gen_name="Gen name",
        action_id="Action id",
        distance="Topological distance"
    )

    @app.callback(
        [Output("inspector_datable", "columns"),
         Output("inspector_datable", "data")],
        [Input('agent_study', 'data'),
         Input("scenario", "data")]
    )
    def update_agent_log_action_table(study_agent, scenario):
        new_episode = make_episode(study_agent, scenario)
        table = actions_model.get_action_table_data(new_episode)
        table['id'] = table['timestep']
        table.set_index('id', inplace=True, drop=False)
        cols_to_exclude = ["id", "lines_modified", "subs_modified", "gens_modified"]
        cols = [{"name": action_table_name_converter[col], "id": col} for col in
                table.columns if col not in cols_to_exclude]
        return cols, table.to_dict("record")

    @app.callback(
        [Output("distribution_substation_action_chart", "figure"),
         Output("distribution_line_action_chart", "figure"),
         Output("distribution_redisp_action_chart", "figure")],
        [Input('agent_study', 'data')],
        [State("distribution_substation_action_chart", "figure"),
         State("distribution_line_action_chart", "figure"),
         State("distribution_redisp_action_chart", "figure"),
         State("scenario", "data"),
         State("agent_ref", "data")]
    )
    def update_agent_log_action_graphs(study_agent, figure_sub, figure_switch_line,
                                       figure_redisp, scenario, ref_agent):
        new_episode = make_episode(study_agent, scenario)
        ref_episode = make_episode(ref_agent, scenario)
        y_max = None
        figure_sub["data"] = actions_model.get_action_per_sub(new_episode)
        if len(figure_sub["data"][0]["x"]) != 0:
            figure_sub["data"].append(actions_model.get_action_per_sub(ref_episode)[0])
            y_max = max(map(max_or_zero, [trace.y for trace in figure_sub["data"]])) + 1
        figure_switch_line["data"] = actions_model.get_action_per_line(new_episode)
        if len(figure_switch_line["data"][0]["x"]) != 0:
            figure_switch_line["data"].append(actions_model.get_action_per_line(ref_episode)[0])
            if y_max is None:
                y_max = max(map(max_or_zero, [trace.y for trace in figure_switch_line["data"]])) + 1
            if max(map(max_or_zero, [trace.y for trace in figure_switch_line["data"]])) > y_max:
                y_max = max(map(max_or_zero, [trace.y for trace in figure_switch_line["data"]])) + 1
        figure_redisp["data"] = actions_model.get_action_redispatch(new_episode)
        if len(figure_redisp["data"][0]["x"]) != 0:
            figure_redisp["data"].append(actions_model.get_action_redispatch(ref_episode)[0])
            if y_max is None:
                y_max = max(map(max_or_zero, [trace.y for trace in figure_redisp["data"]])) + 1
            if max(map(max_or_zero, [trace.y for trace in figure_redisp["data"]])) > y_max:
                y_max = max(map(max_or_zero, [trace.y for trace in figure_redisp["data"]])) + 1

        figure_sub["layout"].update(
            actions_model.update_layout(
                len(figure_sub["data"][0]["x"]) == 0,
                "No Actions on subs for this Agent"))
        figure_switch_line["layout"].update(
            actions_model.update_layout(
                len(figure_switch_line["data"][0]["x"]) == 0,
                "No Actions on lines for this Agent"))
        figure_redisp["layout"].update(
            actions_model.update_layout(
                len(figure_redisp["data"][0]["x"]) == 0,
                "No redispatching actions for this Agent"))

        if y_max:
            figure_sub["layout"]["yaxis"].update(range=[0, y_max])
            figure_switch_line["layout"]["yaxis"].update(range=[0, y_max])
            figure_redisp["layout"]["yaxis"].update(range=[0, y_max])

        return figure_sub, figure_switch_line, figure_redisp

    @app.callback(
        Output("tooltip_table", "children"),
        [Input("inspector_datable", "active_cell")],
        [State('agent_study', 'data'),
         State("scenario", "data"),
         State("inspector_datable", "data")]
    )
    def update_more_info(active_cell, study_agent, scenario, data):
        if active_cell is None:
            raise PreventUpdate
        new_episode = make_episode(study_agent, scenario)
        row_id = active_cell["row_id"]
        act = new_episode.actions[row_id]
        return str(act)


from mrjob.job import MRJob

class MRWordFrequencyCount(MRJob):

    def mapper(self, _, line):
        user,movies,cal,gender,date = line.split(',')
        yield user, float(cal)

    def reducer(self, user, values):
        l = list(values)
        pro = sum(l) / len(l)
        yield user, ("calificacion promedio",pro,"peliculas vistas",len(l))

if __name__ == '__main__':
    MRWordFrequencyCount.run()