import torch.nn as nn
import logging
import torch
logger = logging.getLogger()

class FinalBlock2(nn.Module):
    def __init__(self, opt,in_channels,out_channels) -> None:
        super().__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.in_features = in_channels
        self.out_features = out_channels
        self.fc = nn.Conv1d(in_channels = opt.width,out_channels= opt.num_classes,kernel_size=1,bias=False)
        self.bn = nn.BatchNorm1d(opt.num_classes,affine=opt.affine_bn, eps=opt.bn_eps)
        self.pool = nn.AdaptiveAvgPool1d(1)
    def forward(self,x):
        out =x
        logger.debug(f"before fc x.shape:{out.shape}(opt.width ={self.width} )")
        out = self.fc(out)
        logger.debug(f"after fc x.shape:{out.shape}(num_classes:{self.num_classes})")
        out = out.permute(0,2,1) #B*Lout*num_classes -> B*num_classes*Lout
        logger.debug(f"before bn x.shape:{out.shape}(num_classes:{self.num_classes})")
        out = self.bn(out)
        return out

class RNNNet(nn.Module):
    def __init__(self, opt) -> None:
        super(RNNNet,self).__init__()

        self.num_classes = opt.num_classes
        logger.debug(f"opt.num_classes:{opt.num_classes}")

        self.width = opt.width
        self.initial = nn.Conv1d(4,opt.width,1)# CNN临时大小,opt.width=16
        self.rnn = nn.RNN(opt.width,opt.width,5,batch_first=True)       # RNN临时层数，Watchout
        # self.fc = nn.Linear(in_features = opt.width,out_features =opt.num_classes,bias=False)
        self.fc = FinalBlock2(opt,opt.width,opt.num_classes)
        self.act = nn.ReLU()

    def forward(self,x):
        x = x.permute(0,2,1)
        # logger.debug(f"x.shape:{x.shape}") # B*Lin*4
        # out = self.initial(x.permute(0,2,1))   #B*4*Lin   -> B*Lout*4 -> B*Lout*4
        out = self.initial(x)   #B*4*Lin    -> B*Lout*4
        out = out.permute(0,2,1)#B*16*Lout -> B*Lout*16
        # logger.debug(f"before rnn x.shape:{out.shape}")
        out,_ = self.rnn(out) #B*Lout*16 -> B*Lout*16
        # out = self.pool(out)
        out = out.permute(0,2,1) #B*Lout*num_classes -> B*num_classes*Lout
        # logger.debug(f"before fc x.shape:{out.shape}")
        out = self.fc(out)
        fin_out = self.act(out)
        # logging.debug(f"out shape:{fin_out.shape}")
        return fin_out


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import gast

from tensorflow.python.autograph.core import converter
from tensorflow.python.autograph.pyct import anno
from tensorflow.python.autograph.pyct import ast_util
from tensorflow.python.autograph.pyct import templates
from tensorflow.python.autograph.pyct.static_analysis import annos

class SymbolNamer(object):
  """Describes the interface for ControlFlowTransformer's namer."""

  def new_symbol(self, name_root, reserved_locals):
    """Generate a new unique symbol.

    Args:
      name_root: String, used as stem in the new name.
      reserved_locals: Set(string), additional local symbols that are reserved
          and which should not be used.
    Returns:
      String.
    """
    raise NotImplementedError()

class ControlFlowTransformer(converter.Base):
  """Transforms control flow structures like loops an conditionals."""

  def _create_cond_branch(self, body_name, aliased_orig_names,
                          aliased_new_names, body, returns):
    if not returns:
      # TODO(b/110167197): Replace with a plain return.
      template = """
        return 1
      """
      return_stmt = templates.replace(template)
    elif len(returns) == 1:
      template = """
        return retval
      """
      return_stmt = templates.replace(template, retval=returns[0])
    else:
      template = """
        return (retvals,)
      """
      return_stmt = templates.replace(template, retvals=returns)

    if aliased_orig_names:
      template = """
        def body_name():
          aliased_new_names, = aliased_orig_names,
          body
          return_stmt
      """
      return templates.replace(
          template,
          body_name=body_name,
          body=body,
          aliased_orig_names=aliased_orig_names,
          aliased_new_names=aliased_new_names,
          return_stmt=return_stmt)
    else:
      template = """
        def body_name():
          body
          return_stmt
      """
      return templates.replace(
          template, body_name=body_name, body=body, return_stmt=return_stmt)

  def _create_cond_expr(self, results, test, body_name, orelse_name):
    if results is not None:
      template = """
        results = ag__.if_stmt(test, body_name, orelse_name)
      """
      return templates.replace(
          template,
          test=test,
          results=results,
          body_name=body_name,
          orelse_name=orelse_name)
    else:
      template = """
        ag__.if_stmt(test, body_name, orelse_name)
      """
      return templates.replace(
          template, test=test, body_name=body_name, orelse_name=orelse_name)

  def _fmt_symbols(self, symbol_set):
    if not symbol_set:
      return 'no variables'
    return ', '.join(map(str, symbol_set))

  def _determine_aliased_symbols(self, scope, node_defined_in, block):
    if block:
      block_live_in = set(anno.getanno(block[0], anno.Static.LIVE_VARS_IN))
    else:
      block_live_in = set()

    # For the purpose of aliasing, composite symbols with live owners are live
    # as well. Otherwise this would leak tensors from the conditional's body.
    #
    # For example:
    #
    #   obj = some_obj
    #   if cond:
    #     obj.a = val
    #
    # Thanslating to the code below would be incorrect:
    #
    #   def true_fn():
    #     obj.a = val()  # Wrong! leaks ops owned by true_fn
    #     return obj.a
    for s in scope.modified:
      if s.is_composite():
        live_parents = block_live_in & s.owner_set
        if live_parents:
          block_live_in.add(s)
    return scope.modified & node_defined_in & block_live_in

  def visit_If(self, node):
    body_scope = anno.getanno(node, annos.NodeAnno.BODY_SCOPE)
    orelse_scope = anno.getanno(node, annos.NodeAnno.ORELSE_SCOPE)
    defined_in = anno.getanno(node, anno.Static.DEFINED_VARS_IN)
    live_out = anno.getanno(node, anno.Static.LIVE_VARS_OUT)

    # Note: this information needs to be extracted before the body conversion
    # that happens in the call to generic_visit below, because the conversion
    # generates nodes that lack static analysis annotations.
    need_alias_in_body = self._determine_aliased_symbols(
        body_scope, defined_in, node.body)
    need_alias_in_orelse = self._determine_aliased_symbols(
        orelse_scope, defined_in, node.orelse)

    node = self.generic_visit(node)

    modified_in_cond = body_scope.modified | orelse_scope.modified
    returned_from_cond = set()
    for s in modified_in_cond:
      if s in live_out:
        returned_from_cond.add(s)
      elif s.is_composite():
        # Special treatment for compound objects: if any of their owner entities
        # are live, then they are outputs as well.
        if live_out & s.owner_set:
          returned_from_cond.add(s)

    created_in_body = body_scope.modified & returned_from_cond - defined_in
    created_in_orelse = orelse_scope.modified & returned_from_cond - defined_in

    basic_created_in_body = tuple(
        s for s in created_in_body if not s.is_composite())
    basic_created_in_orelse = tuple(
        s for s in created_in_orelse if not s.is_composite())

    # These variables are defined only in a single branch. This is fine in
    # Python so we pass them through. Another backend, e.g. Tensorflow, may need
    # to handle these cases specially or throw an Error.
    possibly_undefined = (set(basic_created_in_body) ^
                          set(basic_created_in_orelse))

    # Alias the closure variables inside the conditional functions, to allow
    # the functions access to the respective variables.
    # We will alias variables independently for body and orelse scope,
    # because different branches might write different variables.
    aliased_body_orig_names = tuple(need_alias_in_body)
    aliased_orelse_orig_names = tuple(need_alias_in_orelse)
    aliased_body_new_names = tuple(
        self.ctx.namer.new_symbol(s.ssf(), body_scope.referenced)
        for s in aliased_body_orig_names)
    aliased_orelse_new_names = tuple(
        self.ctx.namer.new_symbol(s.ssf(), orelse_scope.referenced)
        for s in aliased_orelse_orig_names)

    alias_body_map = dict(zip(aliased_body_orig_names, aliased_body_new_names))
    alias_orelse_map = dict(
        zip(aliased_orelse_orig_names, aliased_orelse_new_names))

    node_body = ast_util.rename_symbols(node.body, alias_body_map)
    node_orelse = ast_util.rename_symbols(node.orelse, alias_orelse_map)

    cond_var_name = self.ctx.namer.new_symbol('cond', body_scope.referenced)
    body_name = self.ctx.namer.new_symbol('if_true', body_scope.referenced)
    orelse_name = self.ctx.namer.new_symbol('if_false', orelse_scope.referenced)

    returned_from_cond = tuple(returned_from_cond)
    if returned_from_cond:
      if len(returned_from_cond) == 1:
        cond_results = returned_from_cond[0]
      else:
        cond_results = gast.Tuple([s.ast() for s in returned_from_cond], None)

      returned_from_body = tuple(
          alias_body_map[s] if s in need_alias_in_body else s
          for s in returned_from_cond)
      returned_from_orelse = tuple(
          alias_orelse_map[s] if s in need_alias_in_orelse else s
          for s in returned_from_cond)

    else:
      # When the cond would return no value, we leave the cond called without
      # results. That in turn should trigger the side effect guards. The
      # branch functions will return a dummy value that ensures cond
      # actually has some return value as well.
      cond_results = None
      # TODO(mdan): Replace with None once side_effect_guards is retired.
      returned_from_body = (templates.replace_as_expression(
          'ag__.match_staging_level(1, cond_var_name)',
          cond_var_name=cond_var_name),)
      returned_from_orelse = (templates.replace_as_expression(
          'ag__.match_staging_level(1, cond_var_name)',
          cond_var_name=cond_var_name),)

    cond_assign = self.create_assignment(cond_var_name, node.test)
    body_def = self._create_cond_branch(
        body_name,
        aliased_orig_names=aliased_body_orig_names,
        aliased_new_names=aliased_body_new_names,
        body=node_body,
        returns=returned_from_body)
    orelse_def = self._create_cond_branch(
        orelse_name,
        aliased_orig_names=aliased_orelse_orig_names,
        aliased_new_names=aliased_orelse_new_names,
        body=node_orelse,
        returns=returned_from_orelse)
    undefined_assigns = self._create_undefined_assigns(possibly_undefined)

    cond_expr = self._create_cond_expr(cond_results, cond_var_name, body_name,
                                       orelse_name)

    return (undefined_assigns
            + cond_assign
            + body_def
            + orelse_def
            + cond_expr)

  def _create_undefined_assigns(self, undefined_symbols):
    assignments = []
    for s in undefined_symbols:
      template = '''
        var = ag__.Undefined(symbol_name)
      '''
      assignments += templates.replace(
          template,
          var=s,
          symbol_name=gast.Str(s.ssf()))
    return assignments

  def _get_loop_state(self, node):
    body_scope = anno.getanno(node, annos.NodeAnno.BODY_SCOPE)
    defined_in = anno.getanno(node, anno.Static.DEFINED_VARS_IN)
    live_in = anno.getanno(node, anno.Static.LIVE_VARS_IN)
    live_out = anno.getanno(node, anno.Static.LIVE_VARS_OUT)
    reserved_symbols = body_scope.referenced

    # Note that it doesn't matter whether the variables are live after the loop.
    # If the loop modifies them nonlocally (e.g. the result of an iteration
    # depends on the previous iteration), then they need to be included in
    # the loop state, regardless of whether they are later used or not.
    loop_state = body_scope.modified & live_in

    # Variable that are used or defined inside the loop, but not defined
    # before entering the loop
    undefined_lives = ((loop_state - defined_in)
                       | ((body_scope.modified - live_in) & live_out))
    # Only simple variables must be defined. The composite ones will be
    # implicitly checked at runtime.
    undefined_simple_lives = {v for v in undefined_lives if v.is_simple()}

    return loop_state, reserved_symbols, undefined_simple_lives

  def _state_constructs(self, loop_state, reserved_symbols):
    loop_state = list(loop_state)
    state_ssf = [
        self.ctx.namer.new_symbol(s.ssf(), reserved_symbols) for s in loop_state
    ]
    ssf_map = {
        name: ssf
        for name, ssf in zip(loop_state, state_ssf)
        if str(name) != ssf
    }

    state_ast_tuple = gast.Tuple([n.ast() for n in loop_state], None)

    if len(loop_state) == 1:
      loop_state = loop_state[0]
      state_ssf = state_ssf[0]

    return loop_state, state_ssf, state_ast_tuple, ssf_map

  def visit_While(self, node):
    self.generic_visit(node)

    loop_state, reserved_symbols, possibly_undef = self._get_loop_state(node)

    # Note: one might expect we can dispatch based on the loop condition.
    # But because that is dependent on the state, it cannot be evaluated ahead
    # of time - doing that would risk duplicating any effects the condition has.
    # Furthermore, we cannot evaluate slices and attributes, because they might
    # trigger __getitem__ or __getattribute__.
    #
    # A case where this fails includes ops with side effects on a stateful
    # resource captured in an object:
    #
    #   while self.v.read() > 0:
    #     self.v.assign(1)
    #
    # TODO(mdan): Handle the case above.
    cond_scope = anno.getanno(node, annos.NodeAnno.COND_SCOPE)
    cond_closure = set()
    for s in cond_scope.read:
      cond_closure |= s.support_set

    loop_state, state_ssf, state_ast_tuple, ssf_map = self._state_constructs(
        loop_state, reserved_symbols)
    node_body = ast_util.rename_symbols(node.body, ssf_map)
    test = ast_util.rename_symbols(node.test, ssf_map)

    if loop_state:
      template = """
        def test_name(state_ssf):
          return test
        def body_name(state_ssf):
          body
          return state_ssf,
        state_ast_tuple = ag__.while_stmt(
            test_name, body_name, (state,), (extra_deps,))
      """
      node = templates.replace(
          template,
          state=loop_state,
          state_ssf=state_ssf,
          state_ast_tuple=state_ast_tuple,
          test_name=self.ctx.namer.new_symbol('loop_test', reserved_symbols),
          test=test,
          body_name=self.ctx.namer.new_symbol('loop_body', reserved_symbols),
          body=node_body,
          extra_deps=tuple(s.ast() for s in cond_closure),
      )
    else:
      template = """
        def test_name():
          return test
        def body_name():
          body
          return ()
        ag__.while_stmt(test_name, body_name, (), (extra_deps,))
      """
      node = templates.replace(
          template,
          test_name=self.ctx.namer.new_symbol('loop_test', reserved_symbols),
          test=test,
          body_name=self.ctx.namer.new_symbol('loop_body', reserved_symbols),
          body=node_body,
          extra_deps=tuple(s.ast() for s in cond_closure),
      )

    undefined_assigns = self._create_undefined_assigns(possibly_undef)
    return undefined_assigns + node

  def _create_for_loop_early_stopping(self, loop_state, state_ssf,
                                      state_ast_tuple, original_node,
                                      extra_test_name, extra_test,
                                      body_name, loop_body):
    """Create node for for-loop with early stopping (e.g. break or return)."""
    template = """
      def extra_test_name(state_ssf):
        return extra_test_expr
      def body_name(loop_vars, state_ssf):
        # Workaround for PEP-3113
        iterate = loop_vars
        body
        return state_ssf,
      state_ast_tuple = ag__.for_stmt(
          iter_, extra_test_name, body_name, (state,))
    """
    return templates.replace(
        template,
        state=loop_state,
        state_ssf=state_ssf,
        state_ast_tuple=state_ast_tuple,
        iter_=original_node.iter,
        iterate=original_node.target,
        extra_test_name=extra_test_name,
        extra_test_expr=extra_test,
        body_name=body_name,
        body=loop_body)

  def _create_for_loop_with_state(self, loop_state, state_ssf, state_ast_tuple,
                                  original_node, body_name, loop_body):
    """Create node for for-loop with loop-carried state, no early stopping."""
    template = """
      def body_name(loop_vars, state_ssf):
        # Workaround for PEP-3113
        iterate = loop_vars
        body
        return state_ssf,
      state_ast_tuple = ag__.for_stmt(
          iter_, None, body_name, (state,))
    """
    return templates.replace(
        template,
        state=loop_state,
        state_ssf=state_ssf,
        state_ast_tuple=state_ast_tuple,
        iter_=original_node.iter,
        iterate=original_node.target,
        body_name=body_name,
        body=loop_body)

  def _create_for_loop_without_state(self, original_node, body_name, loop_body):
    """Create node for for-loop with loop-carried state, no early stopping."""
    template = """
      def body_name(loop_vars):
        # Workaround for PEP-3113
        iterate = loop_vars
        body
        return ()
      ag__.for_stmt(iter_, None, body_name, ())
    """
    return templates.replace(
        template,
        iter_=original_node.iter,
        iterate=original_node.target,
        body_name=body_name,
        body=loop_body)

  def visit_For(self, node):
    self.generic_visit(node)

    loop_state, reserved_symbols, possibly_undef = self._get_loop_state(node)
    loop_state, state_ssf, state_ast_tuple, ssf_map = self._state_constructs(
        loop_state, reserved_symbols)
    node_body = ast_util.rename_symbols(node.body, ssf_map)
    body_name = self.ctx.namer.new_symbol('loop_body', reserved_symbols)

    has_extra_test = anno.hasanno(node, 'extra_test')
    if loop_state:
      if has_extra_test:
        # Loop with early stopping (e.g. break or return)
        extra_test = anno.getanno(node, 'extra_test')
        extra_test = ast_util.rename_symbols(extra_test, ssf_map)
        extra_test_name = self.ctx.namer.new_symbol('extra_test',
                                                    reserved_symbols)
        node = self._create_for_loop_early_stopping(
            loop_state, state_ssf, state_ast_tuple, node, extra_test_name,
            extra_test, body_name, node_body)
      else:
        # Loop with loop-carried state and no early stopping
        node = self._create_for_loop_with_state(
            loop_state, state_ssf, state_ast_tuple, node, body_name, node_body)
    else:
      # Loop with no loop-carried state and no early stopping
      assert not has_extra_test, ('Early stoppiong (e.g. break and/or return) '
                                  'should create state variables.')
      node = self._create_for_loop_without_state(node, body_name, node_body)

    undefined_assigns = self._create_undefined_assigns(possibly_undef)
    return undefined_assigns + node

def transform(node, ctx):
  node = ControlFlowTransformer(ctx).visit(node)
  return node


import orchestrator.common.xml_tools as xml_tools
import os
from orchestrator.common.logger.maja_logging import configure_logger
from orchestrator.common.maja_exceptions import MajaDataException

LOGGER = configure_logger(__name__)

MUSCATE_HANDLER_XPATH = {
    "Identifier": "//Dataset_Identification/IDENTIFIER",
    "Authority": "//Dataset_Identification/AUTHORITY",
    "Producer": "//Dataset_Identification/PRODUCER",
    "Project": "//Dataset_Identification/PROJECT",
    "AcquisitionDate": "//Product_Characteristics/ACQUISITION_DATE",
    "ProductionDate": "//Product_Characteristics/PRODUCTION_DATE",
    "Instrument": "//Product_Characteristics/INSTRUMENT",
    "SpectralContent": "//Product_Characteristics/SPECTRAL_CONTENT",
    "ProductVersion": "//Product_Characteristics/PRODUCT_VERSION",
    "ZoneGeo": "//GEOGRAPHICAL_ZONE",
    "ProductId": "//Product_Characteristics/PRODUCT_ID",
    "Platform": "//Product_Characteristics/PLATFORM",
    "OrbitNumber": "//Product_Characteristics/ORBIT_NUMBER",
    "UTCAcquisitionRangeMean": "//UTC_Acquisition_Range/MEAN",
    "UTCAcquisitionRangeDatePrecision": "//UTC_Acquisition_Range/DATE_PRECISION",
    "WaterMaskFilename": "//Mask_List/Mask[Mask_Properties/NATURE='Water']/Mask_File_List/MASK_FILE",
    "HiddenSurfaceMaskFilename": "//Mask_List/Mask[Mask_Properties/NATURE='Hidden_Surface']/Mask_File_List/MASK_FILE",
    "TopographyShadowMaskFilename": "//Mask_List/Mask[Mask_Properties/NATURE='Topography_Shadow']/" +
                                    "Mask_File_List/MASK_FILE",
    "SunTooLowMaskFilename": "//Mask_List/Mask[Mask_Properties/NATURE='Sun_Too_Low']/Mask_File_List/MASK_FILE",
    "TangentSunMaskFilename": "//Mask_List/Mask[Mask_Properties/NATURE='Tangent_Sun']/Mask_File_List/MASK_FILE",
    "SnowMaskFilename": "//Mask_List/Mask[Mask_Properties/NATURE='Snow']/Mask_File_List/MASK_FILE",
    "EdgeMaskFilename": "//Mask_List/Mask[Mask_Properties/NATURE='Edge']/Mask_File_List/MASK_FILE",
    "AOTInterpolationMaskFilename": "//Mask_List/Mask[Mask_Properties/NATURE='AOT_Interpolation']/" +
                                    "Mask_File_List/MASK_FILE",
    "PrivateDirectoryFilename": "//Data_List/Data[Data_Properties/NATURE='Private_Directory']/Data_File_List/DATA_FILE",
    "JobProcessingInformationsFileFilename": "//Data_List/Data[Data_Properties/" +
                                             "NATURE='Job_Processing_Informations_File']/Data_File_List/DATA_FILE",
    "DEMInformationMinimum": "/JobProcessingInformations/MNT_Statistics/Min",
    "DEMInformationMaximum": "/JobProcessingInformations/MNT_Statistics/Max",
    "DEMInformationAverage": "/JobProcessingInformations/MNT_Statistics/Average",
    "DEMInformationStandardDeviation": "/JobProcessingInformations/MNT_Statistics/Standard_Deviation",
    "ProcessingFlagsAndModesValidityFlag": "//Processing_Flags_And_Modes_List/" +
                                           "Processing_Flags_And_Modes[Key='Validity_Flag']/Value",
    "ProcessingFlagsAndModesQualityCheck": "//Processing_Flags_And_Modes_List/" +
                                           "Processing_Flags_And_Modes[Key='Quality_check']/Value",
    "ProcessingFlagsAndModesProcessingMode": "//Processing_Flags_And_Modes_List/" +
                                             "Processing_Flags_And_Modes[Key='Processing_Mode']/Value",
    "ProcessingFlagsAndModesProcessingAdjacencyEffectsAndSlopeCorrection": "//Processing_Flags_And_Modes_List/" +
                                    "Processing_Flags_And_Modes[Key='Adjacency_Effects_And_Slope_Correction']/Value",
    "ProcessingFlagsAndModesFileType": "//Processing_Flags_And_Modes_List/" +
                                       "Processing_Flags_And_Modes[Key='File_Type']/Value",
    "ProcessingFlagsAndModesCirrusCorrection": "//Processing_Flags_And_Modes_List/" +
                                               "Processing_Flags_And_Modes[Key='Cirrus_Correction']/Value",
    "ProcessingFlagsAndModesCAMSProcessing": "//Processing_Flags_And_Modes_List/" +
                                             "Processing_Flags_And_Modes[Key='CAMS_Processing']/Value",
    "L1NoData": "//SPECIAL_VALUE[@name='nodata']",
    "Nodata_Value": "//SPECIAL_VALUE[@name='nodata']",
    "VAP_Nodata_Value": "//SPECIAL_VALUE[@name='water_vapor_content_nodata']",
    "AOT_Nodata_Value": "//SPECIAL_VALUE[@name='aerosol_optical_thickness_nodata']",
    "QuantificationValue": "//REFLECTANCE_QUANTIFICATION_VALUE",
    "AerosolOpticalThicknessQuantificationValue": "//AEROSOL_OPTICAL_THICKNESS_QUANTIFICATION_VALUE",
    "AerosolOpticalThicknessQuantificationValueAsString": "//AEROSOL_OPTICAL_THICKNESS_QUANTIFICATION_VALUE",
    "WaterVaporContentQuantificationValue": "//WATER_VAPOR_CONTENT_QUANTIFICATION_VALUE",
    "WaterVaporContentQuantificationValueAsString": "//WATER_VAPOR_CONTENT_QUANTIFICATION_VALUE",
    "Projection": "//Horizontal_Coordinate_System/HORIZONTAL_CS_NAME",
    "ProjectionType": "//Horizontal_Coordinate_System/HORIZONTAL_CS_TYPE",
    "ProjectionCode": "//Horizontal_Coordinate_System/HORIZONTAL_CS_CODE",
    "SunAngleColStep": "//Sun_Angles_Grids/Zenith/COL_STEP",
    "SunAngleRowStep": "//Sun_Angles_Grids/Zenith/ROW_STEP"}

class MuscateXMLFileHandler(object):

    M_SCHEMA_DECIMAL_9_FORMAT = "%.9f"

    def __init__(self, main_xml_file, validate=False, schema_path=None):
        """
        Constructor
        """
        LOGGER.info("Loading " + main_xml_file)
        self.main_xml_file = main_xml_file
        self.root = xml_tools.get_root_xml(self.main_xml_file, deannotate=True)
        if validate and schema_path is not None:
            xml_tools.check_xml(main_xml_file, schema_path)

    # Writing parts
    def save_to_file(self, output_filename):
        # xml_tools.save_to_xml_file(self.root, output_filename)
        xml_tools.save_to_xml_file(self.root, output_filename)
        os.system(
            'XMLLINT_INDENT="    "  xmllint --format ' + str(output_filename) + " --output " + str(output_filename))

    def replace_node(self, node_src, xpath_src, xpath_dst = None, check_if_present=False):
        if xpath_dst is None:
            xml_tools.replace_node(node_src, xpath_src, self.root, xpath_src, check_if_present)
        else:
            xml_tools.replace_node(node_src, xpath_src, self.root, xpath_dst, check_if_present)

    def set_value_of(self, key, value):
        xml_tools.set_value(self.root, MUSCATE_HANDLER_XPATH[key], value)

    def set_string_value(self, xpath, value, check_if_present=True):
        xml_tools.set_value(self.root, xpath, value, check=check_if_present)

    def remove_node(self, xpath):
        xml_tools.remove_node(self.root, xpath)

    # Reading parts

    def get_string_value_of(self, key):
        value = xml_tools.get_only_value(self.root, MUSCATE_HANDLER_XPATH[key])
        if value is not None:
            return value.text
        else:
            raise MajaDataException("No " + MUSCATE_HANDLER_XPATH[key] + " available in the file " + self.main_xml_file)

    def set_validity_flag(self, flag):
        if flag:
            xml_tools.set_value(self.root, MUSCATE_HANDLER_XPATH["ProcessingFlagsAndModesValidityFlag"], "L2VALD")
        else:
            xml_tools.set_value(self.root, MUSCATE_HANDLER_XPATH["ProcessingFlagsAndModesValidityFlag"], "L2NOTV")

    def get_validity_flag(self):
        if "L2VALD" in self.get_string_value_of("ProcessingFlagsAndModesValidityFlag"):
            return True
        else:
            return False

    # Get the upper left corner in latitude / longitude coordinate system
    def get_upper_left_corner(self):
        # upperLeftCorner[0] = Long upperLeftCorner[1] = Lat
        return (
            xml_tools.get_xml_float_value(
                self.root,
                "//Geopositioning/Global_Geopositioning/Point[@name='upperLeft']/LON"),
            xml_tools.get_xml_float_value(
                self.root,
                "//Geopositioning/Global_Geopositioning/Point[@name='upperLeft']/LAT"))

    # Get the upper right corner in latitude / longitude coordinate system
    def get_upper_right_corner(self):
        # upperRightCorner[0] = Long upperRightCorner[1] = Lat
        return (
            xml_tools.get_xml_float_value(
                self.root,
                "//Geopositioning/Global_Geopositioning/Point[@name='upperRight']/LON"),
            xml_tools.get_xml_float_value(
                self.root,
                "//Geopositioning/Global_Geopositioning/Point[@name='upperRight']/LAT"))

    # Get the lower left corner in latitude / longitude coordinate system
    def get_lower_left_corner(self):
        # lowerLeftCorner[0] = Long lowerLeftCorner[1] = Lat
        return (
            xml_tools.get_xml_float_value(
                self.root,
                "//Geopositioning/Global_Geopositioning/Point[@name='lowerLeft']/LON"),
            xml_tools.get_xml_float_value(
                self.root,
                "//Geopositioning/Global_Geopositioning/Point[@name='lowerLeft']/LAT"))

    # Get the lower right corner in latitude / longitude coordinate system
    def get_lower_right_corner(self):
        # lowerRightCorner[0] = Long lowerRightCorner[1] = Lat
        return (
            xml_tools.get_xml_float_value(
                self.root,
                "//Geopositioning/Global_Geopositioning/Point[@name='lowerRight']/LON"),
            xml_tools.get_xml_float_value(
                self.root,
                "//Geopositioning/Global_Geopositioning/Point[@name='lowerRight']/LAT"))

    # Get the center in latitude / longitude coordinate system
    def get_center(self):
        # lowerRightCorner[0] = Long lowerRightCorner[1] = Lat
        return (
            xml_tools.get_xml_float_value(
                self.root,
                "//Geopositioning/Global_Geopositioning/Point[@name='center']/LON"),
            xml_tools.get_xml_float_value(
                self.root,
                "//Geopositioning/Global_Geopositioning/Point[@name='center']/LAT"))

    # Get the mean solar angles
    def get_mean_solar_angles(self):
        # solAngles[0] = zenith solAngles[1] = azimuth
        return (xml_tools.get_xml_float_value(self.root, "//Sun_Angles/ZENITH_ANGLE[@unit='deg']"),
                xml_tools.get_xml_float_value(self.root, "//Sun_Angles/AZIMUTH_ANGLE[@unit='deg']"))

    # Get the mean viewing angles
    def get_mean_viewing_angles(self):
        # vieAngles[0] = zenith vieAngles[1] = azimuth
        if xml_tools.get_only_value(self.root, "//Incidence_Angles/ZENITH_ANGLE[@unit='deg']", check=True) is not None:
            return (xml_tools.get_xml_float_value(self.root, "//Incidence_Angles/ZENITH_ANGLE[@unit='deg']"),
                    xml_tools.get_xml_float_value(self.root, "//Incidence_Angles/AZIMUTH_ANGLE[@unit='deg']"))
        else:
            return 0, 0

    # Get the acquisition date with the YYYYMMDD format
    def get_acquisition_date_formated_yyyymmdd(self):
        ident = xml_tools.get_xml_string_value(self.root, "//Product_Characteristics/ACQUISITION_DATE")
        element = ident.split('T')
        date = element[0]
        return date.replace("-", "")

    # Get the acquisition date in UTC format

    def get_date_pdv_formated_utc(self):
        # Date formated:
        # Case1: <PRODUCTION_DATE>2014-12-17T09:36:57.0</PRODUCTION_DATE>
        # or
        # Case2: <PRODUCTION_DATE>2014-12-17T09:36:57</PRODUCTION_DATE>
        datepdv = xml_tools.get_xml_string_value(self.root, "//Product_Characteristics/ACQUISITION_DATE")
        element = datepdv.split('.')
        msec = ''
        if len(element) == 2:
            msec = element[1]
            if msec[-1] == 'Z':
                msec = msec[:-1]
            msec += '000'
            msec = '.' + msec[0:3] + 'Z'
        date = element[0] + msec
        # Return date formated 'YYYY-MM-DD HH:MM:SS'
        return "UTC=" + date

    def get_mean_viewing_zenithal_angles(self):
        return xml_tools.get_float_list_from_nodes(
            self.root, "//Mean_Viewing_Incidence_Angle_List/Mean_Viewing_Incidence_Angle/ZENITH_ANGLE")

    def get_mean_viewing_azimuthal_angles(self):
        return xml_tools.get_float_list_from_nodes(
            self.root, "//Mean_Viewing_Incidence_Angle_List/Mean_Viewing_Incidence_Angle/AZIMUTH_ANGLE")

    # get the sun zenithal angles
    def get_sun_zenithal_angles(self):
        return xml_tools.get_list_of_string_child_values(self.root, "//Sun_Angles_Grids/Zenith/Values_List")

    # get the sun azimuthal angles
    def get_sun_azimuthal_angles(self):
        return xml_tools.get_list_of_string_child_values(self.root, "//Sun_Angles_Grids/Azimuth/Values_List")

    # get the viewing zenithal angles
    def get_viewing_zenithal_angles(self, bandid):
        if self.has_per_band_angles():
            # S2 case, with per band and per detector angles
            return self._get_list_of_angles_all_detectors_per_band(
                "//Viewing_Incidence_Angles_Grids_List/Band_Viewing_Incidence_Angles_Grids_List", bandid, "Zenith")
        else:
            # Venus case, with per detector angles, 'bandid' is interpreted as detector_id
            zenith_path = "//Viewing_Incidence_Angles_Grids_List/Band_Viewing_Incidence_Angles_Grids_List/Viewing_Incidence_Angles_Grids[@detector_id='{}']/Zenith/Values_List"
            return [xml_tools.get_list_of_string_child_values(self.root,zenith_path.format(bandid))]

    # get the viewing azimuthal angles
    def get_viewing_azimuthal_angles(self, bandid):
        if self.has_per_band_angles():
            # S2 case, with per band and per detector angles
            return self._get_list_of_angles_all_detectors_per_band(
                "//Viewing_Incidence_Angles_Grids_List/Band_Viewing_Incidence_Angles_Grids_List", bandid, "Azimuth")
        else:
            # Venus case, with per detector angles, 'bandid' is interpreted as detector_id
            azimuth_path = "//Viewing_Incidence_Angles_Grids_List/Band_Viewing_Incidence_Angles_Grids_List/Viewing_Incidence_Angles_Grids[@detector_id='{}']/Azimuth/Values_List"
            return [xml_tools.get_list_of_string_child_values(self.root,azimuth_path.format(bandid))]

    def get_viewing_grid_col_step(self, bandid, det):
        if bandid is not None:
            band_selector = "[@band_id='{}']".format(bandid)
        else:
            band_selector = ""
        col_path = "//Viewing_Incidence_Angles_Grids_List/Band_Viewing_Incidence_Angles_Grids_List{}/Viewing_Incidence_Angles_Grids[@detector_id='{}']/Zenith/COL_STEP"
        return xml_tools.get_xml_string_value(self.root,col_path.format(band_selector,det))

    def get_viewing_grid_row_step(self, bandid, det):
        if bandid is not None:
            band_selector = "[@band_id='{}']".format(bandid)
        else:
            band_selector = ""
        row_path = "//Viewing_Incidence_Angles_Grids_List/Band_Viewing_Incidence_Angles_Grids_List{}/Viewing_Incidence_Angles_Grids[@detector_id='{}']/Zenith/ROW_STEP"
        return xml_tools.get_xml_string_value(self.root,row_path.format(band_selector,det))

    def get_viewing_grid_step_unit(self, bandid, det):
        if bandid is not None:
            band_selector = "[@band_id='{}']".format(bandid)
        else:
            band_selector = ""
        zenith_path = "//Viewing_Incidence_Angles_Grids_List/Band_Viewing_Incidence_Angles_Grids_List{}/Viewing_Incidence_Angles_Grids[@detector_id='{}']/Zenith"
        return xml_tools.get_attribute(self.root,zenith_path.format(band_selector,det), "step_unit")

    # Get the list of bands
    def get_list_of_bands(self):
        # Get the number of bands
        l_count = xml_tools.get_attribute(self.root, "//Band_Global_List", "count")

        localpath = "//Band_Global_List[@count='" + str(l_count) + "']/BAND_ID"
        return xml_tools.get_string_list_from_nodes(self.root, localpath)

    # Get the list of bands
    def get_list_of_band_code(self):
        # ---------------------------------------------------------------------------------------------
        localpath = "//Spectral_Band_Informations"
        return xml_tools.get_attribute_values(self.root, localpath, "band_id")

    # Get the index of band
    def get_index_of_band_code(self, band_code):
        # ---------------------------------------------------------------------------------------------
        return self.get_list_of_bands().index(band_code)

    # Get the list of bands for a particuliar resolution
    def get_list_of_band_code_for_resol(self, resol):
        # ---------------------------------------------------------------------------------------------
        l_count = xml_tools.get_attribute(self.root, "//Band_Group_List/Group [@group_id='" + resol + "']/Band_List",
                                          "count")
        localpath = "//Band_Group_List/Group [@group_id='" + resol + "']/Band_List[@count='" + l_count + "']/BAND_ID"
        return xml_tools.get_string_list_from_nodes(self.root, localpath)

    # Get the index of band

    def get_index_of_band_code_for_resol(self, band_code, resol):
        # ---------------------------------------------------------------------------------------------
        return self.get_list_of_band_code_for_resol(resol).index(band_code)

    # Get the saturated mask filename
    def get_list_of_l1_sat_image_filenames(self):
        listofsat = []
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        # Get the list of bands
        listofbands = self.get_list_of_bands()
        # Band Loop
        for bd in listofbands:
            localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Saturation']/Mask_File_List/MASK_FILE[@band_id='"\
                        + bd + "']"
            lsatimagefilename = os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))
            listofsat.append(lsatimagefilename)
        return listofsat

    # Get the saturated mask index within its file
    def get_l1_sat_image_index(self, bandidx):
        # Get the list of bands
        listofbands = self.get_list_of_bands()
        localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Saturation']/Mask_File_List/MASK_FILE[@band_id='" + \
                    listofbands[bandidx] + "']"
        LOGGER.debug("localPath:" + localpath)
        return int(xml_tools.get_attribute(self.root, localpath, "bit_number"))

    def get_list_of_pix_mask_filenames(self):
        listofpix = []
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        # Get the list of bands
        listofbands = self.get_list_of_bands()
        # Band Loop
        for bd in listofbands:
            localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Aberrant_Pixels']/" + \
                        "Mask_File_List/MASK_FILE[@band_id='" + bd + "']"
            lpiximagefilename = os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))
            listofpix.append(lpiximagefilename)
        return listofpix

    def get_list_of_pix_mask_indices(self):
        localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Aberrant_Pixels']/" + \
                "Mask_File_List/MASK_FILE[@band_id='{}']"
        return [int(xml_tools.get_attribute_values(self.root, localpath.format(bd), "bit_number")[0]) for bd in self.get_list_of_bands()]

    def get_l1_pix_mask_index(self, bandidx):
        # Get the list of bands
        listofbands = self.get_list_of_bands()
        localpath = "//Mask[Mask_Properties/NATURE='Aberrant_Pixels']/Mask_File_List/MASK_FILE[@band_id='" + \
            listofbands[bandidx] + "']"
        LOGGER.debug("localPath:" + localpath)
        return [int(f) for f in xml_tools.get_attribute_values(self.root, localpath, "bit_number")]

    # Get the no data mask filename
    def get_list_of_l1_ndt_image_filenames(self):
        listofndt = []
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        # Get the list of bands
        listofbands = self.get_list_of_bands()
        # Band Loop
        for bd in listofbands:
            localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Nodata']/Mask_File_List/MASK_FILE[@band_id='" \
                        + bd + "']"
            lndtimagefilename = os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))
            listofndt.append(lndtimagefilename)
        return listofndt

    # Get the saturated mask index within its file
    def get_l1_ndt_image_index(self, bandidx):
        # Get the list of bands
        listofbands = self.get_list_of_bands()
        localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Nodata']/Mask_File_List/MASK_FILE[@band_id='" + \
            listofbands[bandidx] + "']"
        LOGGER.debug("localPath:" + localpath)
        return int(xml_tools.get_attribute(self.root, localpath, "bit_number"))

    # Get the zone mask filename for the band
    def get_list_of_zones(self, bandid):
        # Convert the band index integer value in string
        localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Detector_Footprint']/Mask_File_List/MASK_FILE[@band_id='"\
                    + bandid + "']"
        # Zones are equivalent to detectors
        # Get the last attribute values (detector index) from nodes
        out = xml_tools.get_attribute_values(self.root, localpath, "detector_id")
        LOGGER.debug("zones (1st try) : "+str(out))
        # if no zone found, try Mean_Viewing_Incidence_Angle (Venus case)
        if (len(out) == 0) and (not self.has_per_band_angles()):
            anglespath = "//Mean_Viewing_Incidence_Angle_List/Mean_Viewing_Incidence_Angle"
            res = self.root.xpath(anglespath)
            LOGGER.debug("nb nodes: "+str(len(res)))
            out = xml_tools.get_attribute_values(self.root, anglespath, "detector_id")
            LOGGER.debug("zones (2nd try) : "+str(out))
        return out

    # Get the zone image index
    def get_l1_dtf_image_index(self, band, detid):
        l_listofbands = self.get_list_of_band_code()
        localpath = "//Mask[Mask_Properties/NATURE='Detector_Footprint']/Mask_File_List/MASK_FILE[@band_id='" + \
            l_listofbands[band] + "'][@detector_id='" + detid + "']"
        LOGGER.debug("localPath:" + localpath)
        return [int(f) for f in xml_tools.get_attribute_values(self.root, localpath, "bit_number")]

    # Get the zone image max index for a detector index
    def get_l1_dtf_max_image_index(self, detid):
        localpath = "//Mask[Mask_Properties/NATURE='Detector_Footprint']/Mask_File_List/MASK_FILE[@detector_id='"\
                    + detid + "']"
        LOGGER.debug("localPath:" + localpath)
        list_of_number = [int(f) for f in xml_tools.get_attribute_values(self.root, localpath, "bit_number")]
        return max(list_of_number)

    # Get the list of name of the L1 TOA image
    def get_list_of_toa_image_filenames(self):
        listofl1toa = []
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        # Get the list of bands
        listofbands = self.get_list_of_bands()
        # Band Loop
        for bd in listofbands:
            localpath = "//IMAGE_FILE[@band_id='" + bd + "']"
            ltoaimagefilename = os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))
            listofl1toa.append(ltoaimagefilename)
        return listofl1toa

    # Get the list of name of the L1 TOA image
    def get_list_of_defective_pixel_image_filenames(self):
        listoffname = []
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        # Get the list of bands
        listofbands = self.get_list_of_bands()
        # Band Loop
        for bd in listofbands:
            localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Defective_Pixel']/" + \
                        "Mask_File_List/MASK_FILE[@band_id='" + bd + "']"
            ldefectimagefilename = os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))
            listoffname.append(ldefectimagefilename)
        return listoffname

    def get_useful_image_infos_filename(self):
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        localpath = "//Data_List/Data[Data_Properties/NATURE='Useful_Image_Informations_File']/Data_File_List/DATA_FILE"
        return os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))

    def get_cla_image_filename(self):
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        localpath = "//Data_List/Data[Data_Properties/NATURE='Cloud_Altitude_Grid']/Data_File_List/DATA_FILE"
        return os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))

    def get_designated_fill_mask_filename(self):
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Designated_Fill']/Mask_File_List/MASK_FILE"
        return os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))

    def get_sol_image_filename(self, p_alt, p_axis):
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        lalt = str(p_alt) + "m"
        localpath = "//Data_List/Data[Data_Properties/NATURE='Solar_Angles_Grid']/Data_File_List/DATA_FILE[@altitude='"\
                    + \
            lalt + "'][@axis='" + p_axis + "']"
        return os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))

    def get_sol_image_index(self, p_alt, p_axis):
        # Get the path in the xml product filename
        lalt = str(p_alt) + "m"
        localpath = "//Data_List/Data[Data_Properties/NATURE='Solar_Angles_Grid']/Data_File_List/DATA_FILE[@altitude='"\
                    + \
            lalt + "'][@axis='" + p_axis + "']"
        return xml_tools.get_attribute_values(self.root, localpath, "band_number")

    def get_vie_image_filename(self, p_det, p_axis):
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        localpath = "//Data_List/Data[Data_Properties/NATURE='Viewing_Angles_Grid']/" + \
                    "Data_File_List/DATA_FILE[@detector_id='" + \
            p_det + "'][@axis='" + p_axis + "']"
        return os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))

    def get_vie_image_index(self, p_det, p_axis):
        # Get the path in the xml product filename
        localpath = "//Data_List/Data[Data_Properties/NATURE='Viewing_Angles_Grid']/" +\
                    "Data_File_List/DATA_FILE[@detector_id='" + \
            p_det + "'][@axis='" + p_axis + "']"
        return xml_tools.get_attribute_values(self.root, localpath, "band_number")

    # Get the list of name of the L1 TOA image
    def get_map_list_of_detector_footprint_image_filenames(self):
        mapofdftname = []
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        # Get the list of bands
        listofbands = self.get_list_of_bands()
        # Band Loop
        for bd in listofbands:
            listofzone = self.get_list_of_zones(bd)
            l_footprintmap = {}
            # detector Loop
            for idz in listofzone:
                localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Detector_Footprint']/" +\
                            "Mask_File_List/MASK_FILE[@band_id='" + bd + "' and @detector_id='" + idz + "']"
                l_imagefilename = os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))
                l_footprintmap[idz] = l_imagefilename
            mapofdftname.append(l_footprintmap)
        return mapofdftname

    def _get_list_of_angles_all_detectors_per_band(self, relativepath, bandid, angletype):
        # AngleType contain "Zenith" or "Azimuth"
        output = []
        # Convert the band index integer value in string
        path = relativepath + "[@band_id='" + bandid + "']/Viewing_Incidence_Angles_Grids"
        zonelist = self.get_list_of_zones(bandid)
        for det in zonelist:
            subpath = path + "[@detector_id='" + det + "']"
            valueslist = xml_tools.get_list_of_string_child_values(
                self.root, os.path.join(subpath, angletype + "/Values_List"))
            output.append(valueslist)
        return output

    """                    L2                      """
    # Get the list of L2 surface reflectance filenames  (Full path)

    def get_list_of_l2_surface_reflectance_filenames(self, resol):
        return self._get_list_of_reflectance_filenames("Surface_Reflectance", resol)

    # Get the list of L2 flat reflectance filenames  (Full path)
    def get_list_of_l2_flat_reflectance_filenames(self, resol):
        return self._get_list_of_reflectance_filenames("Flat_Reflectance", resol)

    def _get_list_of_reflectance_filenames(self, nature, resol):
        list_of_files = []
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        # Get the list of bands
        listofbands = self.get_list_of_band_code_for_resol(resol)
        # Band Loop
        for bd in listofbands:
            localpath = "//Image[Image_Properties/NATURE='" + nature + \
                "']/Image_File_List/IMAGE_FILE[@band_id='" + bd + "']"
            l_imagefilename = os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))
            list_of_files.append(l_imagefilename)

        return list_of_files

    def get_l2_atb_filename(self, resol):
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        localpath = "//Image[Image_Properties/NATURE='Water_Vapor_Content']/Image_File_List/IMAGE_FILE[@group_id='" \
                    + resol + "']"
        return os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))

    def get_l2_cld_filename(self, resol):
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Detailed_Cloud']/Mask_File_List/MASK_FILE[@group_id='" + \
                    resol + "']"
        return os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))

    def get_l2_vap_index(self):
        # Get the path in the xml product filename
        localpath = "//Image[Image_Properties/NATURE='Water_Vapor_Content']/Image_File_List/IMAGE_FILE"
        return xml_tools.get_attribute_values(self.root, localpath, "band_number")

    def get_l2_wat_index(self):
        # Get the path in the xml product filename
        localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Water']/Mask_File_List/MASK_FILE"
        return xml_tools.get_attribute_values(self.root, localpath, "bit_number")

    def get_l2_snow_index(self):
        # Get the path in the xml product filename
        localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Snow']/Mask_File_List/MASK_FILE"
        return xml_tools.get_attribute_values(self.root, localpath, "bit_number")

    def get_l2_aot_index(self):
        # Get the path in the xml product filename
        localpath = "//Image[Image_Properties/NATURE='Aerosol_Optical_Thickness']/Image_File_List/IMAGE_FILE"
        return xml_tools.get_attribute_values(self.root, localpath, "band_number")

    def get_list_of_l2_sat_image_filenames(self, resol):
        listofsat = []
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        # Get the list of bands
        listofbands = self.get_list_of_band_code_for_resol(resol)
        # Band Loop
        for bd in listofbands:
            localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Saturation']/Mask_File_List/MASK_FILE[@band_id='" + \
                        bd + "']"
            lsatimagefilename = os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))
            listofsat.append(lsatimagefilename)
        return listofsat

    def get_l2_edg_filename(self, resol):
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Edge']/Mask_File_List/MASK_FILE[@group_id='" +\
                    resol + "']"
        return os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))

    def get_l2_mg2_filename(self, resol):
        # Get the path in the xml product filename
        lpath = os.path.dirname(self.main_xml_file)
        localpath = "//Mask_List/Mask[Mask_Properties/NATURE='Water']/Mask_File_List/MASK_FILE[@group_id='" + \
                    resol + "']"
        return os.path.join(lpath, xml_tools.get_xml_string_value(self.root, localpath))

    def get_the_xpath_root_of_the_private_directory_filename(self):
        return "//Data_List/Data[Data_Properties/NATURE='Private_Directory']"

    def has_per_band_angles(self):
        nodes = self.root.xpath("//Mean_Value_List/Mean_Viewing_Incidence_Angle_List/Mean_Viewing_Incidence_Angle")
        if len(nodes):
            if 'band_id' in nodes[0].keys():
                return True
        return False


from paypal.standard.ipn.signals import valid_ipn_received
from django.dispatch import receiver
from django.shortcuts import get_object_or_404
from .models import CausesDonation, ProjectDonation
from django.core.mail import send_mail
from django.template.loader import render_to_string

@receiver(valid_ipn_received)
def payment_notification(sender, **kwargs):
    ipn = sender
    if ipn.payment_status == 'Completed':
        donation = None
        if ipn.custom == 'cause':
            donation = get_object_or_404(CausesDonation, uuid=ipn.invoice)
        elif ipn.custom == 'project':
            donation = get_object_or_404(ProjectDonation, uuid=ipn.invoice)
        if donation.amount == ipn.mc_gross:
            donation.status = 'Paid'
            donation.save()
            send_mail(
                subject="Mico Foundation Donation Receipt",
                from_email="info@themicofoundationja.com",
                recipient_list=[donation.email, ],
                message='Donation Receipt',
                html_message=render_to_string('donation_receipt.html', {
                    'donation': donation,
                }),
                fail_silently=True,
            )


import numpy as np
from keras.models import load_model
import io
from keras.preprocessing.image import img_to_array
from PIL import Image
from keras.models import model_from_json

file_json = 'selfie_model.json'
h5_file = 'selfie_model-weights.h5'
#model = load_model("selfie_model.h5")
read_json = open(file_json, 'r')
loaded_model_json = read_json.read()
read_json.close()
model= model_from_json(loaded_model_json)
model.load_weights(h5_file)

image = Image.open('Positive-(1).jpg')
image = image.resize((256,256), Image.ANTIALIAS)
image = image.convert("L")

imgar = img_to_array(image)
imgar = np.expand_dims(imgar, axis=0)

pred = model.predict(imgar)
#classes = model.predict_classes(imgar)
#label = classes[0]
label = np.argmax(pred)
digit = np.max(pred)
if label == 0:
    name = "fix"
elif label == 1:
    name = "not"
else:
    name = "other"
prediction = {"prediction":{'label':label, 'probability':digit}, "class":name}

print(prediction)


import xmltodict
import copy
import sys

def readFile(name):
    file = open(name,'r',encoding='utf-8')
    readed = file.read()
    file.close()
    return readed

def to_dict(text):
    return xmltodict.parse(text)

# Recursively find key in dictionary
def finditem(obj, key):
    if key in obj: return obj[key]
    for k, v in obj.items():
        if isinstance(v,dict):
            item = finditem(v, key)
            if item is not None:
                return item

gCount = 0
# Iterate through each level, create an array and pass back up
def forEach(obj, group):
    if isinstance(obj,str):
        return []
    global gCount
    gCount += 1
    # Get initial information, also check that each of the keys exist or there are issues
    entry = {}
    entry['group'] = group
    entry['@level'] = obj['@level']
    # Check for otherlevel
    if '@otherlevel' in obj:
        entry['@otherlevel'] = obj['@otherlevel']
    if 'unitid' in obj['did'].keys():
        # Get rid of occurences of 'Series'
        entry['unitid'] = obj['did']['unitid']
        # If we don't end with a period, add one for Excel to not get confused
        if entry['unitid'] is not None:
            entry['unitid'] = entry['unitid'].replace("Series ",'')
            if entry['unitid'][-1] != '.':
                entry['unitid'] += '.'
            # Then, add another one because excel is still confused
            entry['unitid'] += '.'
    else:
        entry['unitid'] = -1
    if 'unittitle' in obj['did'].keys() and '#text' in obj['did']['unittitle'].keys():
        entry['unittitle'] = obj['did']['unittitle']['#text']
    else:
        entry['unittitle'] = ""

    # Clean up newlines in the middle of the title
    entry['unittitle'] = entry['unittitle'].replace('\n','')
    while (entry['unittitle'].count('  ') != 0):
        entry['unittitle'] = entry['unittitle'].replace('  ',' ')

    # Finish creating entry if we are not a series
    if (obj['@level'] != 'series'):
        # Check that we have the container key
        if 'container' in obj['did'].keys():
            # Check that we only have 2
            if type(obj['did']['container']) is list:
                # Get box and folder number out, assuming it exists
                temp = obj['did']['container']
                for i in temp:
                    if '#text' in i.keys():
                        # Replace the dash so Excel doesn't read the cell as a date
                        entry[i['@type'].lower()] = i['#text'].replace("-"," and ")
            # Otherwise, we only have one so use that
            else:
                entry[obj['did']['container']['@type']] = obj['did']['container']['#text']
        # Get item count if we have scopecontent
        if 'scopecontent' in obj.keys():
            entry['itemCount'] = obj['scopecontent']['p']
            # Digital Commons things
            if entry['itemCount'] is not None and not isinstance(entry['itemCount'],str):
                entry['digitalCommons'] = copy.deepcopy(entry['itemCount'])
                if '#text' in entry['digitalCommons'].keys():
                    # Put things in correct indices
                    del entry['digitalCommons']['#text']
                    entry['itemCount'] = entry['itemCount']['#text'].split("(")[0]
                    # Get all links for digital commons into an array
                    links = []
                    # Check if we have an array or a dictionary
                    if isinstance(entry['digitalCommons']['extref'], list):
                        for l in entry['digitalCommons']['extref']:
                            links.append( l['@xlink:href'] )
                    else:
                        links.append( entry['digitalCommons']['extref']['@xlink:href'] )
                    entry['digitalCommons'] = links
                else:
                    entry['itemCount'] = ""
            # Get rid of extra information from text
            if isinstance(entry['itemCount'],str):
                entry['itemCount'] = ''.join([i for i in entry['itemCount'] if i.isdigit()])

    # Recurse to get all children
    entries = []
    for k,v in obj.items():
        if 'c0' in k:
            if isinstance(v,dict):
                for e in forEach(v,group):
                    entries.append(e)
            elif isinstance(v,list):
                for i in v:
                    for e in forEach(i,group):
                        entries.append(e)

    # Keep track of all entries
    entries.append(entry)
    return entries

# Create TSV file
def sendToFile(name, entries):
    out = ""
    keys = ['group','@level','@otherlevel','box','folder','unitid','unittitle','itemCount','digitalCommons']
    # Create the header at the top
    for i in keys:
        out += i + "\t"
    out += "\n"
    # Add all entries to output
    for e in entries:
        for i in keys:
            if i in e.keys():
                if isinstance(e[i],str):
                    out += e[i]
                elif e[i] is not None:
                    for j in e[i]:
                        out += j + '\t'
            out += '\t'
        out += '\n'

    file = open(name,'w')
    file.write(out)
    file.close()

def main():
    # Get all source files
    file = open("sources.txt",'r')
    sources = file.read().split()
    file.close()

    allEntries = []

    for i in sources:
        print(i)
        # Read file and convert to dictionary
        parsed = to_dict(readFile(i))

        # Get the tables out
        found = finditem(parsed, 'dsc')
        table = found['c01']

        ret = []
        if isinstance(table, list):
            for j in table:
                ret = forEach(j,i)
                for e in ret:
                    allEntries.append(e)
        else:
            ret = forEach(table,i)
            for e in ret:
                allEntries.append(e)

    print("Total found:",len(allEntries))
    print("Expected entries:",gCount)
    print()

    sendToFile("output.tsv",allEntries)

main()


#!/usr/bin/env python
'''auxopt command handling'''

import time, os

def name():
    '''return module name'''
    return "auxopt"

def description():
    '''return module description'''
    return "auxopt command handling"

def init(_mpstate):
    '''initialise module'''
    global mpstate
    mpstate = _mpstate
    mpstate.command_map['auxopt'] = (cmd_auxopt,   'select option for aux switches on CH7 and CH8 (ArduCopter only)')
    mpstate.completions['auxopt'] = ['set <7|8> <Nothing|Flip|SimpleMode|RTL|SaveTrim|SaveWP|MultiMode|CameraTrigger|Sonar|Fence|ResetYaw|SuperSimpleMode|AcroTrainer|Acro|Auto|AutoTune|Land>',
                                     'reset <7|8|all>',
                                     '<show|list>']

aux_options = {
    "Nothing":"0",
    "Flip":"2",
    "SimpleMode":"3",
    "RTL":"4",
    "SaveTrim":"5",
    "SaveWP":"7",
    "MultiMode":"8",
    "CameraTrigger":"9",
    "Sonar":"10",
    "Fence":"11",
    "ResetYaw":"12",
    "SuperSimpleMode":"13",
    "AcroTrainer":"14",
    "Auto":"16",
    "AutoTune":"17",
    "Land":"18"
}

def aux_show(channel):
    param = "CH%s_OPT" % channel
    opt_num = str(int(mpstate.functions.get_mav_param(param)))
    option = None
    for k in aux_options.keys():
        if opt_num == aux_options[k]:
            option = k
            break
    else:
        print("AUX Channel is currently set to unknown value " + opt_num)
        return
    print("AUX Channel is currently set to " + option)

def aux_option_validate(option):
    for k in aux_options:
        if option.upper() == k.upper():
            return k
    return None

def cmd_auxopt(args):
    '''handle AUX switches (CH7, CH8) settings'''
    if mpstate.vehicle_type != 'copter':
        print("This command is only available for copter")
        return
    if len(args) == 0 or args[0] not in ('set', 'show', 'reset', 'list'):
        print("Usage: auxopt set|show|reset|list")
        return
    if args[0] == 'list':
        print("Options available:")
        for s in sorted(aux_options.keys()):
            print('  ' + s)
    elif args[0] == 'show':
        if len(args) > 2 and args[1] not in ['7', '8', 'all']:
            print("Usage: auxopt show [7|8|all]")
            return
        if len(args) < 2 or args[1] == 'all':
            aux_show('7')
            aux_show('8')
            return
        aux_show(args[1])
    elif args[0] == 'reset':
        if len(args) < 2 or args[1] not in ['7', '8', 'all']:
            print("Usage: auxopt reset 7|8|all")
            return
        if args[1] == 'all':
            mpstate.functions.param_set('CH7_OPT', '0')
            mpstate.functions.param_set('CH8_OPT', '0')
            return
        param = "CH%s_OPT" % args[1]
        mpstate.functions.param_set(param, '0')
    elif args[0] == 'set':
        if len(args) < 3 or args[1] not in ['7', '8']:
            print("Usage: auxopt set 7|8 OPTION")
            return
        option = aux_option_validate(args[2])
        if not option:
            print("Invalid option " + args[2])
            return
        param = "CH%s_OPT" % args[1]
        mpstate.functions.param_set(param, aux_options[option])
    else:
        print("Usage: auxopt set|show|list")


from collections import namedtuple
import csv
import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm
from sklearn.decomposition import PCA
from sklearn.metrics import auc, roc_curve, precision_recall_curve
from varclr.models.model import Encoder
# Ref: https://scikit-learn.org/stable/auto_examples/svm/plot_oneclass.html

embeddings = Encoder.from_pretrained("varclr-codebert")

# def train_svm(train, normal, outliers):
def train_svm(train: list[str], test: list[tuple[str, bool]], sink: str):
    # print(train)
    # print(test[:5])
    X_train = embeddings.encode(train)
    # X_test = embeddings.encode([f for f, _ in test])
    # fit the model
    clf = svm.OneClassSVM(nu=1e-2, kernel="rbf", gamma=0.05, tol=1e-3, verbose=False)
    clf.fit(X_train)
    return clf

def read_logging_flows(path: str):
    flows_dict = {}
    with open(path, 'r') as f:
        reader = csv.reader(f)
        headers = next(reader)
        for row in reader:
            flow = namedtuple('Flow', ['name', 'sink', 'label'])
            flow.name = row[0]
            flow.sink = row[1]
            flow.label = row[2]
            flows_dict[flow] = flow.label
    return flows_dict

def get_stat(all_flows, scores: list, optimal_threshold: float):
    tp = 0
    fp = 0
    fn = 0
    tn = 0
    for i, (name, label) in enumerate(all_flows):
        if (label == 'FP' or label == 'TN') and scores[i]>=optimal_threshold:
            fp += 1
            # print('fp', name)
            # print(scores[i])
        elif (label == 'TP' or label == 'FN') and scores[i]>=optimal_threshold:
            tp += 1
            # print('tp', name)
            # print(scores[i])
        elif (label == 'FP' or label == 'TN') and scores[i]<optimal_threshold:
            tn += 1
            # print('tn', name)
            # print(scores[i])
        elif (label == 'TP' or label == 'FN') and scores[i]<optimal_threshold:
            fn += 1
            # print('fn', name)
            # print(scores[i])
    # double check
    # print('Positive', len([i for i in scores if i>=optimal_threshold]))
    # print('Negative', len([i for i in scores if i<optimal_threshold]))
    print('tp',tp)
    print('fp',fp)
    print('fn',fn)
    print('tn',tn)

if __name__ == '__main__':
    samples = {
        'logging': {
            'normal': ['authkey', 'password', 'passcode', 'passphrase'],
            'unusual': []
        },
    }
    ground_truth = {}
    scores = {}
    # ground_truth_spec_dict = {}
    # not_sensitive_spec_dict = {}
    dataset_total_path = 'data/logging_flows_ground_truth.csv'
    dataset_not_sensitive_path = 'data/logging_flows_not_sensitive_unique.csv'
    all_flows_spec_dict = read_logging_flows(dataset_total_path)
    not_sensitive_spec_dict = read_logging_flows(dataset_not_sensitive_path)

    sinks_to_test = sorted(set([spec.sink for spec in all_flows_spec_dict.keys()]))
    train_size_list = []
    f1_list = []
    loop_for_self_label = 1
    for i in range(loop_for_self_label):
        for sink in sinks_to_test:
            all_flows = [(spec.name, label) for spec, label in all_flows_spec_dict.items() if spec.sink == sink]
            clf = train_svm(
                [name for name in samples[sink]['normal']],
                all_flows,
                sink
            )
            all_flows_emb = embeddings.encode([f for f, _ in all_flows])
            scores[sink] = clf.score_samples(all_flows_emb)

            ## No need to flip, as we want to catch the "normal" cases (i.e. words similar to "password", etc.)
            # normalize and flip the sign of the scores (in OneClassSVM, a low score means an outlier)
            # scores[sink] = [-float(i)/max(scores[sink]) for i in scores[sink]]

            ground_truth[sink] = [1 if label == 'TP' or label == 'FN' else 0 for _, label in all_flows]

        colors = {
            'logging': 'red',
            'NoSkill': 'blue'
        }
        # Plot ROC and AUC score
        plt.figure(figsize=(10, 8))
        plt.rcParams.update({'font.size': 20})
        color = iter(plt.cm.rainbow(np.linspace(
            0, 1, len(sinks_to_test))))
        plt.plot([0, 1], [0, 1], linestyle='--', label='NoSkill', color=colors['NoSkill'])
        for sink in sinks_to_test:
            c = next(color)
            y_true = ground_truth[sink]
            y_score = scores[sink]

            fpr, tpr, thresholds = roc_curve(y_true, y_score)
            roc_auc = auc(fpr, tpr)

            plt.plot(fpr, tpr, marker='.', label=f"ROC of {sink} (AUC = {roc_auc:.2f})", color=c)
        lw = 2
        plt.plot([0, 1], [0, 1], color="navy", lw=lw, linestyle="--")
        plt.xlim([0.0, 1.0])
        plt.ylim([0.0, 1.05])
        plt.title(f"Novelty Detection\nReceiver operating characteristic of logging")
        plt.xlabel('False Positive Rate')
        plt.ylabel('True Positive Rate')
        plt.legend(loc="lower right")
        # plt.show()
        plt.savefig('oc_svm_logging_roc.pdf', bbox_inches='tight')
        plt.clf()

        # Plot PR curve and F1-score
        color = iter(plt.cm.rainbow(np.linspace(
            0, 1, len(sinks_to_test))))
        for sink in sinks_to_test:
            c = next(color)
            y_true = ground_truth[sink]
            y_score = scores[sink]

            precision, recall, thresholds = precision_recall_curve(y_true, y_score)
            f1 = np.divide(2*precision*recall, precision+recall,
                            out=np.zeros(precision.shape, dtype=float), where=(precision+recall) != 0)
            optimal_idx = np.argmax(f1)
            optimal_threshold = thresholds[optimal_idx]
            optimal_precision = precision[optimal_idx]
            optimal_recall = recall[optimal_idx]
            optimal_f1 = (2*optimal_precision*optimal_recall)/(optimal_precision+optimal_recall)
            print(f"\nOptimal (best F1-score) threshold of {sink} is {optimal_threshold}")
            print(f"Precision of {sink}'s optimal threshold is {optimal_precision}")
            print(f"Recall of {sink}'s optimal threshold is {optimal_recall}")
            print(f"F1-score of {sink}'s optimal threshold is {optimal_f1}")
            plt.plot(recall, precision, marker='.', label=f"{sink} (F1-score={optimal_f1:.2f})", color=c)
            train_size_list.append(len(samples[sink]['normal']))
            f1_list.append(optimal_f1)

            # Find FNs in non-sensitive flows
            # not_sensitive_flows = [(spec.name, label) for spec, label in not_sensitive_spec_dict.items() if spec.sink == sink]
            # not_sensitive_flows_emb = embeddings.encode([f for f, _ in not_sensitive_flows])
            # not_sensitive_flows_score = clf.score_samples(not_sensitive_flows_emb)
            # get_stat(not_sensitive_flows, not_sensitive_flows_score, optimal_threshold)

            # Code for Debugging
            if sink == 'logging':
                all_flows = [(spec.name, label) for spec, label in all_flows_spec_dict.items() if spec.sink == sink]
                get_stat(all_flows, scores[sink], optimal_threshold)

        lw = 2
        # plt.plot([0, 1], [0, 1], color="navy", lw=lw, linestyle="--")
        plt.xlim([0.0, 1.0])
        plt.ylim([0.0, 1.05])
        plt.title(f"Novelty Detection\nPrecision-Recall curve of logging")
        plt.xlabel('Recall')
        plt.ylabel('Precision')
        plt.legend(loc="lower right")
        # plt.show()
        plt.savefig('oc_svm_logging_pr.pdf', bbox_inches='tight')

    # print('train_size_list', train_size_list)
    # print('f1_list', f1_list)


import tkinter

# ウィンドウ作成
root = tkinter.Tk()
root.title("先生にしつもん")
root.minsize(640,480)
root.option_add("*font",["MS Pゴシック,22"])

#画像表示
canvas=tkinter.Canvas(bg="green",width=640,height=480)
canvas.place(x=0,y=0)
img=tkinter.PhotoImage(file="chap3-back.png")
canvas.create_image(320,240,image=img)

#テキスト表示
question=tkinter.Label(text="計算機実習２",bg="light blue")
question.place(x=100,y=40)

#テキストボックスを表示
entry=tkinter.Entry(width=12,bd=4)
entry.place(x=50,y=133)

#質問ボタン表示
askbutton=tkinter.Button(text="聞く")
askbutton.place(x=260,y=125)

#答え表示
answer=tkinter.Label(text="・・・・・・",bg="white")
answer.place(x=115,y=235)

#イベント設定
def ask_click():
    print("ボタンがクリックされました！")

askbutton["command"]=ask_click

# メインループ
root.mainloop()


import torch
import torch.nn as nn
import statistics
import torchvision.models as models

class EncoderCNN(nn.Module):
    def __init__(self, embed_size):
        super(EncoderCNN, self).__init__()
        self.encoder = models.wide_resnet50_2(pretrained=True)
        self.encoder.fc = nn.Linear(self.encoder.fc.in_features, embed_size)
        self.relu = nn.ReLU()
        self.times = []
        self.dropout = nn.Dropout(0.5)

    def forward(self, images, test=False):
        # if test:
        #     features = self.encoder(images)
        # else:
        #     features, _ = self.encoder(images)
        features = self.encoder(images)
        features = self.relu(features)
        features = self.dropout(features)

        return features

class DecoderRNN(nn.Module):
    def __init__(self, embed_size, hidden_size, vocab_size, num_layers):
        super(DecoderRNN, self).__init__()
        self.embed = nn.Embedding(vocab_size, embed_size)
        self.lstm = nn.LSTM(embed_size, hidden_size, num_layers)
        self.linear = nn.Linear(hidden_size, vocab_size)
        self.dropout = nn.Dropout(0.5)

    def forward(self, features, captions):
        embeddings = self.embed(captions)  # [word_length, batch_size, embed_size]
        embeddings = self.dropout(embeddings)  # features [batch_size, embed_size]
        embeddings = torch.cat((features.unsqueeze(0), embeddings), dim=0)  # [word_length, batch_size, embed_size]
        hiddens, _ = self.lstm(embeddings)
        outputs = self.linear(hiddens)

        return outputs

class CNNtoRNN(nn.Module):
    def __init__(self, cfg):
        super(CNNtoRNN, self).__init__()
        self.embed_size = cfg.EMBED_SIZE
        self.hidden_size = cfg.HIDDEN_SIZE
        self.vocab_size = cfg.VOCAB_SIZE
        self.num_layers = cfg.NUM_LAYERS

        self.encoderCNN = EncoderCNN(self.embed_size)
        self.decoderRNN = DecoderRNN(self.embed_size, self.hidden_size, self.vocab_size, self.num_layers)

    def forward(self, images, captions):
        features = self.encoderCNN(images)
        outputs = self.decoderRNN(features, captions)
        return outputs

    def caption_image(self, image, vocabulary, max_length=50):
        result_caption = []

        with torch.no_grad():
            x = self.encoderCNN(image, test=True).unsqueeze(0)
            states = None

            for _ in range(max_length):
                hiddens, states = self.decoderRNN.lstm(x, states)
                output = self.decoderRNN.linear(hiddens.squeeze(0))

                predicted = output.argmax(1)
                result_caption.append(predicted.item())

                x = self.decoderRNN.embed(predicted).unsqueeze(0)

                if vocabulary.itos[predicted.item()] == "<EOS>":
                    break

        return [vocabulary.itos[idx] for idx in result_caption]


# --*-- built-in packages --*--
import logging
import logging.handlers
import sys

# --*-- Installed packages --*--

# --*-- own packages --*--

def create_log(log_name, level = logging.INFO):

    # # create logger
    if len(sys.argv)==2:
        if sys.argv[1].lower() == "info":
            level = logging.INFO
        elif sys.argv[1].lower() == "debug":
            level = logging.DEBUG
        elif sys.argv[1].lower() == "warning":
            level = logging.WARNING
        else:
            print("Argument not recognized...")
            print("Setting DEBUG level")
            level = logging.DEBUG

    # # logger = logging.getLogger(__name__)
    logger = logging.getLogger(log_name)
    logger.setLevel(level)
    # # create formatter
    detailed_formatter = logging.Formatter(
        "%(asctime)s %(name)s[%(process)d] %(funcName)s(): "+\
        "%(levelname)s - %(message)s")

    #  # create a console handler
    # # and set its log level to the command-line option
    # #
    console_handler = logging.StreamHandler(sys.stdout)
    console_handler.setLevel(level)
    console_handler.setFormatter(detailed_formatter)

    # # create a file handler
    # # and set its log level to DEBUG
    # #
    file_handler = logging.handlers.TimedRotatingFileHandler(
        f'logs/{log_name}.log', 'midnight', 1, 30)
    file_handler.setLevel(level)
    file_handler.setFormatter(detailed_formatter)

    # # add handlers to the "mypackage" logger
    # #
    logger.addHandler(console_handler)
    logger.addHandler(file_handler)

    return logger


import pygame
import random
import time
import sys
from pygame.locals import *
from Ball import Ball
from Board import Board
from Brick import Brick
import configparser

class Game(object):
    config = configparser.ConfigParser()
    config.read("config.ini")

    directionBall = dict(config['config'])

    BGCOLOR = (47, 79, 79)
    BLACK = (0, 0, 0)
    WHITE = (255, 255, 255)
    BLUE = (0, 0, 255)
    GREEN = (0, 255, 0)
    RED = (255, 0, 0)
    LIGHTYELLOW = (255, 204, 0)
    LIST_BRICK = []

    def __init__(self):
        pygame.init()
        self.screen = pygame.display.set_mode((680, 500))
        pygame.display.set_caption("Breakout Game!")
        pygame.mouse.set_visible(False)
        self.clock = pygame.time.Clock()
        self.board = Board(200, self.screen.get_height()-40, 100, 20)
        Game.LIST_BRICK = self.createBricks(70, 23, 90)
        self.ball = Ball(self.screen.get_width() // 2, (self.screen.get_height() // 2)+150, Game.directionBall["d1"], 1, 1, 10, self.screen.get_width(), self.screen.get_height(), Game.LIST_BRICK)

        self.pause = False
        self.score = 0

    def run(self):
        while True:
            if self.pause:
                self.pauseGame()
                continue

            self.clock.tick(280)
            for event in pygame.event.get():
                if event.type == QUIT:
                    pygame.quit()
                    sys.exit()
                if event.type == KEYDOWN:
                    if event.key == K_ESCAPE:
                        pygame.quit()
                        sys.exit()
                    if event.key == K_SPACE:
                        if self.pause:
                            self.pause = False
                        else:
                            self.pause = True

            #print(self.score)
            self.ball.move(Game)
            self.board.move()
            self.draw()

    def draw(self):
        self.screen.fill(Game.BGCOLOR)
        self.ball.draw(self.screen)
        self.board.draw(self.screen)
        for brick in Game.LIST_BRICK:
            brick.draw(self.screen)
        pygame.display.update()

    def createBricks(self, width, height, n):
        list_brick = []
        posX = 0
        posY = 30
        distBorda = 25

        x = 0
        z = width
        while z > distBorda:
            z -= 1
            x -= 1

        for i in range(1, n+1):
            if i == 1:
                posX += x + width
            else:
                posX += width

            if (posX + width) > self.screen.get_width() - distBorda:
                posY += height
                posX = distBorda

            brick = Brick(posX, posY, width, height)
            list_brick.append(brick)
        return list_brick

    def pauseGame(self):
        event = pygame.event.wait()
        if event.type == K_SPACE:
            self.pause = False


from logging import Logger
from pathlib import Path
from typing import List, Tuple, Any, Optional, Dict

import numpy as np
import optuna
import pandas as pd
from optuna import Trial
from optuna.samplers import TPESampler
from sklearn.pipeline import Pipeline

from python import DOCUMENT_ID, EVENT_ID, SENTENCE_IDX, TOKEN_IDX_FROM, TOKEN_IDX_TO
from python.handwritten_baseline.pipeline.model.scripts.evaluation_utils import run_conll_evaluation
from python.handwritten_baseline import LEMMA
from python.handwritten_baseline.pipeline.model.data_prep.pipeline_data_input import get_X_and_y_for_pipeline, \
    create_gold_clustering
from python.handwritten_baseline.pipeline.model.scripts.baselines.doc_clustering import create_doc_clustering_pipeline
from python.handwritten_baseline.pipeline.model.scripts.scoring import obtain_final_coref_metrics
from python.handwritten_baseline.pipeline.model.scripts.train_predict_optimize import load_data
from python.pipeline import RUN_WORKING_DIR
from python.util.optuna import PlotCallback

LEMMA_BASELINE = "lemma"
LEMMA_WD_BASELINE = "lemma-wd"
LEMMA_DELTA_BASELINE = "lemma-delta"
LEMMA_TIME_BASELINE = "lemma-time"

def predict_baseline(logger: Logger,
                     X: List[Tuple],
                     y: List[Any],
                     which: str,
                     doc_clustering_pipeline: Optional[Pipeline] = None) -> Tuple[pd.Series, pd.Series]:
    """

    :param logger:
    :param X:
    :param y:
    :param which: which baseline to run
    :param doc_clustering_pipeline: only used for lemma-delta
    :return: tuple of gold clustering and predicted cluster
    """
    assert len(X) == 1 and len(y) == 1

    dataset, documents, mentions = X[0]
    mentions_to_gold_events = y[0]

    # if a document clustering pipeline is given, apply it, otherwise put all documents in the same cluster
    if which in [LEMMA_DELTA_BASELINE, LEMMA_TIME_BASELINE]:
        if doc_clustering_pipeline is None:
            raise ValueError
        doc_clustering = doc_clustering_pipeline.predict(X)
        logger.info(f"Document clustering created {len(doc_clustering[EVENT_ID].unique())} cluster(s).")
    else:
        docs = documents[DOCUMENT_ID].values
        doc_clustering = pd.DataFrame({DOCUMENT_ID: docs, EVENT_ID: pd.Series(np.zeros_like(docs, dtype=np.int))})

    # obtain lemma for each action phrase, creating those in a per-document fashion is easiest
    mention_lemmas_in_docs = []
    for doc_id, df in mentions.groupby(DOCUMENT_ID):
        lemmas_in_doc = dataset.tokens.loc[(doc_id), LEMMA]

        mention_lemmas_in_doc = df.apply(lambda row: " ".join(lemmas_in_doc.loc[row[SENTENCE_IDX], slice(row[TOKEN_IDX_FROM], row[TOKEN_IDX_TO] - 1)]), axis=1)

        # for the lemma-wd baseline, make sure we have unique mention lemmas in each document by prepending the doc id
        if which == LEMMA_WD_BASELINE:
            mention_lemmas_in_doc = doc_id + "__" + mention_lemmas_in_doc

        mention_lemmas_in_docs.append(mention_lemmas_in_doc)
    mention_lemmas_in_docs = pd.concat(mention_lemmas_in_docs)

    # confine mention lemmas to document clusters by prepending the cluster id to each lemma
    for cluster_id, doc_ids in doc_clustering.groupby(EVENT_ID)[DOCUMENT_ID]:
        # remove documents which do not have mentions
        doc_ids = doc_ids.loc[doc_ids.isin(mention_lemmas_in_docs.index.get_level_values(DOCUMENT_ID))]
        mention_lemmas_in_docs.loc[doc_ids] = str(cluster_id) + "__" + mention_lemmas_in_docs.loc[doc_ids]

    # don't let the name of the method fool you into thinking there is gold data involved here, it just does exactly
    # the thing we want to do here
    system_clusters = create_gold_clustering(mention_lemmas_in_docs)
    gold_clusters = create_gold_clustering(mentions_to_gold_events)

    # Our clustering approach cannot produce multiple clusters for one mention. We still need to map the integer
    # cluster identifiers in lists.
    gold_clusters = gold_clusters.map(lambda v: [v])
    system_clusters = system_clusters.map(lambda v: [v])

    return gold_clusters, system_clusters

def optimize_thresh(train_X,
                    train_y,
                    signal: str,
                    min_thresh: float,
                    max_thresh: float,
                    logger: Logger,
                    serialization_dir: Path,
                    n_trials: int = 50):
    # Optimize the clustering thresh for tunable baselines: We fit a doc clustering pipeline on the training split, then
    # we predict document clusters on train, use those to run lemma delta and evaluate with LEA F1 against the gold
    # event coref clusters. Barhom et al. optimized the document clustering performance itself, we optimize document
    # clustering + coref which is more faithful to the actual task and should provide fairer, more comparable results
    # across multiple datasets.
    def objective(trial: Trial):
        delta = trial.suggest_float("delta", min_thresh, max_thresh)
        pipeline = create_doc_clustering_pipeline(delta, signal)
        pipeline.fit(train_X)

        gold_clusters, system_clusters = predict_baseline(logger, train_X, train_y,
                                                          which=LEMMA_DELTA_BASELINE,
                                                          doc_clustering_pipeline=pipeline)

        metrics = run_conll_evaluation(gold_clusters, system_clusters, single_meta_document=True, metrics="lea")
        lea_f1 = metrics.loc["lea", "f1"]
        return lea_f1

    sampler = TPESampler(seed=1)
    study = optuna.create_study(sampler=sampler, direction="maximize")
    study.optimize(objective, n_trials=n_trials, callbacks=[PlotCallback(serialization_dir=serialization_dir / signal / "plots")])
    best_delta = study.best_trial.params["delta"]
    return best_delta

def run_baselines(config_data: Dict,
                  config_baselines: List,
                  config_global: Dict,
                  logger: Logger) -> None:
    """
    Runs lemma baseline, lemma-delta baseline on the given dataset and writes scores to `serialization_dir`.
    The delta hyperparameter for lemma-delta is optimized by training on train and evaluating on dev.
    :param config_data
    :param config_baselines
    :param config_global
    :param logger
    """

    serialization_dir = config_global[RUN_WORKING_DIR]

    # load datasets
    def load_split(p):
        data = load_data(p)
        X, y = get_X_and_y_for_pipeline(logger,
                                        data,
                                        doc_partitioning=None,
                                        oracle_mention_pair_generation=False)
        return X, y

    train_X, train_y = load_split(config_data["train_data_path"])
    eval_X, eval_y = load_split(config_data["eval_data_path"])

    lemma_delta_best_thresh = None
    if LEMMA_DELTA_BASELINE in config_baselines:
        logger.info("Starting lemma-delta threshold optimization.")
        lemma_delta_best_thresh = optimize_thresh(train_X, train_y, "tfidf", 0.0, 1.0, logger, serialization_dir)
        logger.info(f"Best thresh for lemma-delta: {lemma_delta_best_thresh}")

    lemma_time_best_thresh = None
    if LEMMA_TIME_BASELINE in config_baselines:
        logger.info("Starting lemma-time threshold optimization.")
        lemma_time_best_thresh = optimize_thresh(train_X, train_y, "time", 0.0, 10 * 365 * 24, logger, serialization_dir)
        logger.info(f"Best thresh for lemma-time: {lemma_time_best_thresh}")

    evaluation_results_dir = serialization_dir / "results"
    evaluation_results_dir.mkdir(exist_ok=True)
    for baseline_name in config_baselines:
        logger.info(f"Predicting for {baseline_name}")
        if baseline_name == LEMMA_DELTA_BASELINE:
            # fit tfidf on eval
            pipeline = create_doc_clustering_pipeline(lemma_delta_best_thresh, signal="tfidf")
            pipeline.fit(eval_X)
        elif baseline_name == LEMMA_TIME_BASELINE:
            pipeline = create_doc_clustering_pipeline(lemma_time_best_thresh, signal="time")
        else:
            pipeline = None

        gold_clusters, system_clusters = predict_baseline(logger, eval_X, eval_y,
                                                          which=baseline_name,
                                                          doc_clustering_pipeline=pipeline)

        dir_of_baseline = evaluation_results_dir / baseline_name
        dir_of_baseline.mkdir(exist_ok=True)

        logger.info(f"Computing metrics for {baseline_name}")
        obtain_final_coref_metrics(gold_clusters,
                                   system_clusters,
                                   f"eval_{baseline_name}",
                                   "all",
                                   serialization_dir=dir_of_baseline)


import os
import time
from tempfile import mkdtemp
from unittest.mock import MagicMock
from mujoco_py.builder import enable_compiler_caching

def touch(file_path):
    with open(file_path, 'w') as touched_file:
        touched_file.write('foo')

def artifact(temp_dir, artifact_name):
    return os.path.join(temp_dir, artifact_name)

def mock_compiler_side_effect(obj, src, ext, cc_args, extra_postargs, pp_opts):
    touch(obj)

def mock_compiler(temp_dir, setup):
    compiler = MagicMock()
    compiler.compiler_type = 'unix'
    compiler._get_cc_args = MagicMock(return_value='')
    compiler._setup_compile = MagicMock(return_value=(
        None, setup.keys(), None, None, setup))
    compiler._compile = MagicMock(side_effect=mock_compiler_side_effect)
    return compiler

def test_caching_compiler():
    temp_dir = mkdtemp()

    a_o = artifact(temp_dir, 'a.o')
    a_c = artifact(temp_dir, 'a.c')
    a_o_meta = artifact(temp_dir, 'a.o.meta')
    touch(a_c)

    b_o = artifact(temp_dir, 'b.o')
    b_cc = artifact(temp_dir, 'b.cc')
    b_o_meta = artifact(temp_dir, 'b.o.meta')
    touch(b_cc)

    compiler = mock_compiler(temp_dir, {a_o : (a_c, 'c'), b_o : (b_cc, 'cc')})
    enable_compiler_caching(compiler)

    compiler.compile(None)
    assert compiler._compile.call_count is 2

    compiler.compile(None)
    assert compiler._compile.call_count is 2

    os.remove(a_o)
    os.remove(b_o_meta)
    compiler.compile(None)
    assert compiler._compile.call_count is 4

    time.sleep(0.1)
    touch(a_c)
    compiler.compile(None)
    assert compiler._compile.call_count is 5


from dataclasses import dataclass
import math

@dataclass
class Document:
    id: str
    relevance: float

@dataclass
class QueryWord:
    data: str
    is_minus: bool
    is_stop: bool

@dataclass
class Query:
    plus_words: set
    minus_words: set

class SearchEngine:
    MAX_RESULT_DOCUMENT_COUNT: int
    _stop_words: set
    _word_to_document_freqs: dict
    _documents: set

    def __init__(self, max_result_document_count: int = 5):
        self.MAX_RESULT_DOCUMENT_COUNT = max_result_document_count
        self._stop_words = set()
        self._word_to_document_freqs = dict()
        self._documents = set()

    # public methods
    def set_stop_words(self, text: str):
        self._stop_words = set(text.split())

    def add_document(self, document_id: str, document: str):
        document = document.lower().replace('\r\n', ' ').replace('\n', ' ')
        words = self.__split_into_words_no_stop(document)
        if len(words) > 0:
            inv_word_count = 1 / len(words)
            for word in words:
                if word not in self._word_to_document_freqs.keys():
                    self._word_to_document_freqs[word] = dict()
                if document_id in self._word_to_document_freqs[word].keys():
                    self._word_to_document_freqs[word][document_id] += inv_word_count
                else:
                    self._word_to_document_freqs[word][document_id] = inv_word_count
            self._documents.add(document_id)

    def find_top_documents(self, raw_query: str):
        raw_query = raw_query.lower()
        query = self.__parse_query(raw_query)
        matched_documents = self.__find_all_documents(query)
        matched_documents.sort(key=lambda x: x.relevance, reverse=True)
        if len(matched_documents) > self.MAX_RESULT_DOCUMENT_COUNT:
            return matched_documents[:self.MAX_RESULT_DOCUMENT_COUNT]
        return matched_documents

    def get_document_count(self):
        return len(self._documents)

    def match_document(self, raw_query: str, document_id: str):
        query = self.__parse_query(raw_query)
        matched_words = list()
        for word in query.plus_words:
            for key in self._word_to_document_freqs.keys():
                if word in key:
                    if document_id in self._word_to_document_freqs[key].keys():
                        matched_words.append(word)
                    break
        for word in query.minus_words:
            if word in self._word_to_document_freqs.keys():
                if document_id in self._word_to_document_freqs[word].keys():
                    matched_words.clear()
        return matched_words

        # private methods

    def __is_stop_word(self, word: str) -> bool:
        return word in self._stop_words

    def __split_into_words_no_stop(self, text: str) -> list:
        return [word for word in text.split() if not self.__is_stop_word(word)]

    def __parse_query_word(self, text: str) -> QueryWord:
        is_minus = False
        """
        if text[0] == '-':
            is_minus = True
            text = text[1:]
        """
        return QueryWord(text, is_minus, self.__is_stop_word(text))

    def __parse_query(self, text: str) -> Query:
        query = Query(set(), set())
        for word in text.split():
            query_word = self.__parse_query_word(word)
            if not query_word.is_stop:
                if query_word.is_minus:
                    query.minus_words.add(query_word.data)
                else:
                    query.plus_words.add(query_word.data)
        return query

    def __compute_word_inverse_document_freq(self, word: str) -> float:
        if word in self._word_to_document_freqs.keys():
            return math.log(self.get_document_count() / len(self._word_to_document_freqs[word]))
        else:
            for key in self._word_to_document_freqs.keys():
                if word in key:
                    return math.log(self.get_document_count() / len(self._word_to_document_freqs[key]))

    def __find_all_documents(self, query: Query) -> list:
        document_to_relevance = dict()
        for word in query.plus_words:
            if word in self._word_to_document_freqs.keys():
                inverse_document_freq = self.__compute_word_inverse_document_freq(word)
                for document_id, term_freq in self._word_to_document_freqs[word].items():
                    if document_id in document_to_relevance.keys():
                        document_to_relevance[document_id] += term_freq * inverse_document_freq
                    else:
                        document_to_relevance[document_id] = term_freq * inverse_document_freq
            else:
                for key in self._word_to_document_freqs.keys():
                    if word in key:
                        inverse_document_freq = self.__compute_word_inverse_document_freq(word)
                        for document_id, term_freq in self._word_to_document_freqs[key].items():
                            if document_id in document_to_relevance.keys():
                                document_to_relevance[document_id] += term_freq * inverse_document_freq * 0.8
                            else:
                                document_to_relevance[document_id] = term_freq * inverse_document_freq * 0.8
                        break

        for word in query.minus_words:
            if word in self._word_to_document_freqs.keys():
                for document_id in self._word_to_document_freqs[word].keys():
                    del self._word_to_document_freqs[word][document_id]
        matched_documents = list()
        for document_id, relevance in document_to_relevance.items():
            matched_documents.append(Document(document_id, relevance))
        return matched_documents

def print_document(document: Document):
    print(f"id = {document.id}, relevance = {document.relevance}")

def search_documents(documents: list, query: str, max_result_document_count: int = 5) -> list:
    result = list()
    if max_result_document_count < 0:
        max_result_document_count = len(documents)
    search_engine = SearchEngine(max_result_document_count)
    for document in documents:
        search_engine.add_document(str(document.id), document.content)
    print(query)
    for document in search_engine.find_top_documents(query):
        print_document(document)
        result.append(document.id)
    del search_engine
    return result