# coding: utf-8

from __future__ import absolute_import, division, print_function

import logging
import time
from collections import defaultdict
from math import ceil, cos, floor, log, pi, sin, sqrt

from dials.array_family import flex
from dials.util.version import dials_version
from iotbx import mtz
from libtbx.utils import Sorry
from scitbx import matrix

try:
  from math import isclose
except ImportError:
  # Python 3 backport
  def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
      return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)

logger = logging.getLogger(__name__)

def sum_partial_reflections(integrated_data, min_total_partiality=0.5):
  '''Sum partial reflections; weighted sum for summation integration; weighted
  average for profile fitted reflections. N.B. this will report total
  partiality for the summed reflection.'''

  if not 'partiality' in integrated_data:
    return integrated_data

  # rest of the columns are pretty well defined - these are all uniform for the
  # reflection so can just ignore other values:
  #
  # d
  # id
  # entering
  # flags
  # lp
  # miller_index
  # panel
  # partial_id
  # s1
  # xyzcal.mm
  # xyzcal.px
  # zeta
  #
  # Then finally these need a weighted average:
  #
  # intensity.prf.value
  # intensity.prf.variance
  # intensity.sum.value
  # intensity.sum.variance
  # partiality
  # bbox - this is not used...
  #
  # now just need to worry about those that I am actually outputting to the MTZ
  # file...

  isel = (integrated_data['partiality'] < 0.99).iselection()

  if len(isel) == 0:
    return integrated_data

  delete = flex.size_t()
  partial_map = defaultdict(list)

  # create map of partial_id to reflections j

  for j in isel:
    partial_map[integrated_data['partial_id'][j]].append(j)

  # now work through this map - get total partiality for every reflection;
  # here only consider reflections with > 1 component; if total partiality
  # less than min_total_partiality discard all parts.

  partial_ids = []

  for p_id in partial_map:
    if len(partial_map[p_id]) > 1:
      partial_ids.append(p_id)

  # work through multipart partials; compute those weighted values I need
  # if total partiality less than min, delete. if summing, delete extra parts

  we_got_profiles = 'intensity.prf.value' in integrated_data
  logger.info('Profile fitted reflections: %s' % we_got_profiles)

  for p_id in partial_ids:
    p_tot = sum([integrated_data['partiality'][j] for j in partial_map[p_id]])
    if p_tot < min_total_partiality:
      for j in partial_map[p_id]:
        delete.append(j)
      continue

    j0 = partial_map[p_id][0]
    jrest = partial_map[p_id][1:]

    # FIXME revisiting this calculation am not sure it is correct - why
    # weighting by (I/sig(I))^2 not just 1/variance?
    if we_got_profiles:
      prf_value = integrated_data['intensity.prf.value'][j0]
      prf_variance = integrated_data['intensity.prf.variance'][j0]
      weight = prf_value * prf_value / prf_variance
      prf_value *= weight
      prf_variance *= weight
      total_weight = weight
    sum_value = integrated_data['intensity.sum.value'][j0]
    sum_variance = integrated_data['intensity.sum.variance'][j0]
    partiality = integrated_data['partiality'][j0]

    # weight profile fitted intensity and variance computed from weights
    # proportional to (I/sig(I))^2; schedule for deletion spare parts

    for j in jrest:
      delete.append(j)

      sum_value += integrated_data['intensity.sum.value'][j]
      sum_variance += integrated_data['intensity.sum.variance'][j]
      partiality += integrated_data['partiality'][j]

      if we_got_profiles:
        _prf_value = integrated_data['intensity.prf.value'][j]
        _prf_variance = integrated_data['intensity.prf.variance'][j]

        _weight = _prf_value * _prf_value / _prf_variance
        prf_value += _weight * _prf_value
        prf_variance += _weight * _prf_variance
        total_weight += _weight

    # now write these back into original reflection
    if we_got_profiles:
      prf_value /= total_weight
      prf_variance /= total_weight
      integrated_data['intensity.prf.value'][j0] = prf_value
      integrated_data['intensity.prf.variance'][j0] = prf_variance
    integrated_data['intensity.sum.value'][j0] = sum_value
    integrated_data['intensity.sum.variance'][j0] = sum_variance
    integrated_data['partiality'][j0] = partiality

  integrated_data.del_selected(delete)

  return integrated_data

def scale_partial_reflections(integrated_data, min_partiality=0.5):
  '''Scale partial reflections (after summation) according to their estimated
  partiality - for profile fitted reflections this will result in no change,
  for summation integrated reflections will be scaled up by 1 / partiality
  with error accordingly scaled. N.B. this will report the scaled up partiality
  for the output reflection.'''

  # assert: in here there will be no multi-part partial reflections

  if not 'partiality' in integrated_data:
    return integrated_data

  isel = (integrated_data['partiality'] < 1.0).iselection()

  if len(isel) == 0:
    return integrated_data

  delete = flex.size_t()

  for j in isel:
    if integrated_data['partiality'][j] < min_partiality:
      delete.append(j)
      continue
    inv_p = 1.0 / integrated_data['partiality'][j]
    integrated_data['intensity.sum.value'][j] *= inv_p
    integrated_data['intensity.sum.variance'][j] *= inv_p
    integrated_data['partiality'][j] *= 1.0

  integrated_data.del_selected(delete)

  return integrated_data

def dials_u_to_mosflm(dials_U, uc):
  '''Compute the mosflm U matrix i.e. the U matrix from same UB definition
  as DIALS, but with Busing & Levy B matrix definition.'''

  parameters = uc.parameters()
  dials_B = matrix.sqr(uc.fractionalization_matrix()).transpose()
  dials_UB = dials_U * dials_B

  r_parameters = uc.reciprocal_parameters()

  a = parameters[:3]
  al = [pi * p / 180.0 for p in parameters[3:]]
  b = r_parameters[:3]
  be = [pi * p / 180.0 for p in r_parameters[3:]]

  mosflm_B = matrix.sqr((b[0], b[1] * cos(be[2]), b[2] * cos(be[1]),
                         0, b[1] * sin(be[2]), - b[2] * sin(be[1]) * cos(al[0]),
                         0, 0, 1.0 / a[2]))

  mosflm_U = dials_UB * mosflm_B.inverse()

  return mosflm_U

def _apply_data_filters(integrated_data,
                        ignore_profile_fitting, filter_ice_rings, min_isigi,
                        include_partials, keep_partials, scale_partials):
  """Apply filters to reflection data"""

  # select reflections that are assigned to an experiment (i.e. non-negative id)
  integrated_data = integrated_data.select(integrated_data['id'] >= 0)
  assert len(integrated_data), "No experiment-assigned reflections"
  logger.info('Read %s predicted reflections' % len(integrated_data))

  # Ignore profile fitted
  if ignore_profile_fitting:
    del integrated_data['intensity.prf.value']
    del integrated_data['intensity.prf.variance']

  if 'intensity.prf.variance' in integrated_data:
    selection = integrated_data.get_flags(
      integrated_data.flags.integrated,
      all=True)
  else:
    selection = integrated_data.get_flags(
      integrated_data.flags.integrated_sum)
  integrated_data = integrated_data.select(selection)
  logger.info("Selected %d integrated reflections" % len(integrated_data))

  # check we have reflections left - see #357
  if len(integrated_data) == 0:
    if ignore_profile_fitting:
      raise Sorry("All reflections excluded based on flags.integrated")
    else:
      raise Sorry("No profile fitted reflections, "
                  "please try ignore_profile_fitting=True")

  selection = integrated_data['intensity.sum.variance'] <= 0
  if selection.count(True) > 0:
    integrated_data.del_selected(selection)
    logger.info('Removing %d reflections with negative variance' % \
          selection.count(True))

  if 'intensity.prf.variance' in integrated_data:
    selection = integrated_data['intensity.prf.variance'] <= 0
    if selection.count(True) > 0:
      integrated_data.del_selected(selection)
      logger.info('Removing %d profile reflections with negative variance' % \
            selection.count(True))

  if filter_ice_rings:
    selection = integrated_data.get_flags(integrated_data.flags.in_powder_ring)
    integrated_data.del_selected(selection)
    logger.info("Removing %d reflections in ice ring resolutions" %
                selection.count(True))

  if min_isigi is not None:

    selection = (
      integrated_data['intensity.sum.value']/
      flex.sqrt(integrated_data['intensity.sum.variance'])) < min_isigi
    integrated_data.del_selected(selection)
    logger.info('Removing %d reflections with I/Sig(I) < %s' %(
      selection.count(True), min_isigi))

    if 'intensity.prf.variance' in integrated_data:
      selection = (
        integrated_data['intensity.prf.value'] /
        flex.sqrt(integrated_data['intensity.prf.variance'])) < min_isigi
      integrated_data.del_selected(selection)
      logger.info('Removing %d profile reflections with I/Sig(I) < %s' %(
        selection.count(True), min_isigi))

  # FIXME in here work on including partial reflections => at this stage best
  # to split off the partial refections into a different selection & handle
  # gracefully... better to work on a short list as will need to "pop" them &
  # find matching parts to combine.

  if include_partials:
    integrated_data = sum_partial_reflections(integrated_data)
    if scale_partials:
      integrated_data = scale_partial_reflections(integrated_data)

  if 'partiality' in integrated_data:
    selection = integrated_data['partiality'] < 0.99
    if selection.count(True) > 0 and not keep_partials:
      integrated_data.del_selected(selection)
      logger.info('Removing %d incomplete reflections' % \
        selection.count(True))

  return integrated_data

def _add_batch(mtz, experiment, batch_number, image_number, force_static_model):
  """Add a single image's metadata to an mtz file.

  Returns the batch object.
  """
  assert batch_number > 0

  # Recalculate useful numbers and references here
  wavelength = experiment.beam.get_wavelength()
  # We ignore panels beyond the first one, at the moment
  panel = experiment.detector[0]

  if experiment.goniometer:
    axis = matrix.col(experiment.goniometer.get_rotation_axis())
  else:
    axis = 0.0, 0.0, 0.0

  U = matrix.sqr(experiment.crystal.get_U())
  if experiment.goniometer is not None:
    F = matrix.sqr(experiment.goniometer.get_fixed_rotation())
  else:
    F = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))

  # Create the batch object and start configuring it
  o = mtz.add_batch().set_num(batch_number).set_nbsetid(1).set_ncryst(1)
  o.set_time1(0.0).set_time2(0.0).set_title('Batch {}'.format(batch_number))
  o.set_ndet(1).set_theta(flex.float((0.0, 0.0))).set_lbmflg(0)
  o.set_alambd(wavelength).set_delamb(0.0).set_delcor(0.0)
  o.set_divhd(0.0).set_divvd(0.0)

  # FIXME hard-coded assumption on indealized beam vector below... this may be
  # broken when we come to process data from a non-imgCIF frame
  s0n = matrix.col(experiment.beam.get_s0()).normalize().elems
  o.set_so(flex.float(s0n)).set_source(flex.float((0, 0, -1)))

  # these are probably 0, 1 respectively, also flags for how many are set, sd
  o.set_bbfac(0.0).set_bscale(1.0)
  o.set_sdbfac(0.0).set_sdbscale(0.0).set_nbscal(0)

  # unit cell (this is fine) and the what-was-refined-flags FIXME hardcoded

  # take time-varying parameters from the *end of the frame* unlikely to
  # be much different at the end - however only exist if scan-varying
  # refinement was used
  if not force_static_model and experiment.crystal.num_scan_points > 0:
    # Get the index of the image in the sequence e.g. first => 0, second => 1
    image_index = image_number - experiment.image_range[0]
    _unit_cell = experiment.crystal.get_unit_cell_at_scan_point(image_index)
    _U = matrix.sqr(experiment.crystal.get_U_at_scan_point(image_index))
  else:
    _unit_cell = experiment.crystal.get_unit_cell()
    _U = U

  # apply the fixed rotation to this to unify matrix definitions - F * U
  # was what was used in the actual prediction: U appears to be stored
  # as the transpose?! At least is for Mosflm...
  #
  # FIXME Do we need to apply the setting rotation here somehow? i.e. we have
  # the U.B. matrix assuming that the axis is equal to S * axis_datum but
  # here we are just giving the effective axis so at scan angle 0 this will
  # not be correct... FIXME 2 not even sure we can express the stack of
  # matrices S * R * F * U * B in MTZ format?... see [=A=] below
  _U = dials_u_to_mosflm(F * _U, _unit_cell)

  # FIXME need to get what was refined and what was constrained from the
  # crystal model - see https://github.com/dials/dials/issues/355
  o.set_cell(flex.float(_unit_cell.parameters()))
  o.set_lbcell(flex.int((-1, -1, -1, -1, -1, -1)))
  o.set_umat(flex.float(_U.transpose().elems))

  # get the mosaic spread though today it may not actually be set - should
  # this be in the BATCH headers?
  try:
    mosaic = experiment.crystal.get_mosaicity()
  except AttributeError:
    mosaic = 0
  o.set_crydat(flex.float([mosaic, 0.0, 0.0, 0.0, 0.0, 0.0,
                           0.0, 0.0, 0.0, 0.0, 0.0, 0.0]))

  o.set_lcrflg(0)
  o.set_datum(flex.float((0.0, 0.0, 0.0)))

  # detector size, distance
  o.set_detlm(flex.float([0.0, panel.get_image_size()[0],
                          0.0, panel.get_image_size()[1],
                          0, 0, 0, 0]))
  o.set_dx(flex.float([panel.get_directed_distance(), 0.0]))

  # goniometer axes and names, and scan axis number, and num axes, missets
  # [=A=] should we be using this to unroll the setting matrix etc?
  o.set_e1(flex.float(axis))
  o.set_e2(flex.float((0.0, 0.0, 0.0)))
  o.set_e3(flex.float((0.0, 0.0, 0.0)))
  o.set_gonlab(flex.std_string(('AXIS', '', '')))
  o.set_jsaxs(1)
  o.set_ngonax(1)
  o.set_phixyz(flex.float((0.0, 0.0, 0.0, 0.0, 0.0, 0.0)))

  # scan ranges, axis
  if experiment.scan:
    phi_start, phi_range = experiment.scan.get_image_oscillation(image_number)
  else:
    phi_start, phi_range = 0.0, 0.0

  o.set_phistt(phi_start)
  o.set_phirange(phi_range)
  o.set_phiend(phi_start + phi_range)
  o.set_scanax(flex.float(axis))

  # number of misorientation angles
  o.set_misflg(0)

  # crystal axis closest to rotation axis (why do I want this?)
  o.set_jumpax(0)

  # type of data - 1; 2D, 2; 3D, 3; Laue
  o.set_ldtype(2)

  return o

def _write_columns(mtz_file, dataset, integrated_data, scale_partials,
                   apply_scales):
  """Write the column definitions AND data for a single dataset."""

  # now create the actual data structures - first keep a track of the columns

  # H K L M/ISYM BATCH I SIGI IPR SIGIPR FRACTIONCALC XDET YDET ROT WIDTH
  # LP MPART FLAG BGPKRATIOS

  # gather the required information for the reflection file

  nref = len(integrated_data['miller_index'])

  # check reflections remain
  if nref == 0:
    raise Sorry('no reflections for export')

  xdet, ydet, zdet = [flex.double(x) for x in integrated_data['xyzobs.px.value'].parts()]

  # compute BATCH values - floor() to get (fortran) image captured within
  #                        +1     because FORTRAN counting; zdet+1=image_index
  #                        +off   because            image_index+o=batch
  batch = (flex.floor(zdet).iround() + 1) + integrated_data["batch_offset"]

  # we're working with full reflections so... #388 no longer guaranteed
  if scale_partials:
    fractioncalc = flex.double(nref, 1.0)
  else:
    fractioncalc = integrated_data['partiality']

  # now add column information...

  # FIXME add DIALS_FLAG which can include e.g. was partial etc.

  type_table = {
    'H': 'H',
    'K': 'H',
    'L': 'H',
    'I': 'J',
    'SIGI': 'Q',
    'IPR': 'J',
    'SIGIPR': 'Q',
    'BG' : 'R',
    'SIGBG' : 'R',
    'XDET': 'R',
    'YDET': 'R',
    'BATCH': 'B',
    'BGPKRATIOS': 'R',
    'WIDTH': 'R',
    'MPART': 'I',
    'M_ISYM': 'Y',
    'FLAG': 'I',
    'LP': 'R',
    'FRACTIONCALC': 'R',
    'ROT': 'R',
    'QE': 'R',
  }

  # derive index columns from original indices with
  #
  # from m.replace_original_index_miller_indices
  #
  # so all that is needed now is to make space for the reflections - fill with
  # zeros...

  mtz_file.adjust_column_array_sizes(nref)
  mtz_file.set_n_reflections(nref)

  # assign H, K, L, M_ISYM space
  for column in 'H', 'K', 'L', 'M_ISYM':
    dataset.add_column(column, type_table[column]).set_values(
      flex.double(nref, 0.0).as_float())

  mtz_file.replace_original_index_miller_indices(integrated_data['miller_index_rebase'])

  dataset.add_column('BATCH', type_table['BATCH']).set_values(
    batch.as_double().as_float())

  if 'lp' in integrated_data:
    lp = integrated_data['lp']
  else:
    lp = flex.double(nref, 1.0)
  if 'qe' in integrated_data:
    qe = integrated_data['qe']
  elif 'dqe' in integrated_data:
    qe = integrated_data['dqe']
  else:
    qe = flex.double(nref, 1.0)
  I_profile = None
  V_profile = None
  I_sum = None
  V_sum = None
  # FIXME errors in e.g. LP correction need to be propagated here
  scl = lp / qe

  if apply_scales:
    scl = scl / integrated_data['inverse_scale_factor']

  if 'intensity.prf.value' in integrated_data:
    I_profile = integrated_data['intensity.prf.value'] * scl
    V_profile = integrated_data['intensity.prf.variance'] * scl * scl
    # Trap negative variances
    assert V_profile.all_gt(0)
    dataset.add_column('IPR', type_table['I']).set_values(I_profile.as_float())
    dataset.add_column('SIGIPR', type_table['SIGI']).set_values(
      flex.sqrt(V_profile).as_float())
  if 'intensity.sum.value' in integrated_data:
    I_sum = integrated_data['intensity.sum.value'] * scl
    V_sum = integrated_data['intensity.sum.variance'] * scl * scl
    # Trap negative variances
    assert V_sum.all_gt(0)
    dataset.add_column('I', type_table['I']).set_values(I_sum.as_float())
    dataset.add_column('SIGI', type_table['SIGI']).set_values(
      flex.sqrt(V_sum).as_float())
  if ('background.sum.value' in integrated_data and
      'background.sum.variance' in integrated_data):
    bg = integrated_data['background.sum.value']
    varbg = integrated_data['background.sum.variance']
    assert (varbg >= 0).count(False) == 0
    sigbg = flex.sqrt(varbg)
    dataset.add_column('BG', type_table['BG']).set_values(bg.as_float())
    dataset.add_column('SIGBG', type_table['SIGBG']).set_values(sigbg.as_float())

  dataset.add_column('FRACTIONCALC', type_table['FRACTIONCALC']).set_values(
    fractioncalc.as_float())

  dataset.add_column('XDET', type_table['XDET']).set_values(xdet.as_float())
  dataset.add_column('YDET', type_table['YDET']).set_values(ydet.as_float())
  dataset.add_column('ROT', type_table['ROT']).set_values(integrated_data["ROT"].as_float())
  dataset.add_column('LP', type_table['LP']).set_values(lp.as_float())
  dataset.add_column('QE', type_table['QE']).set_values(qe.as_float())

def _next_epoch(val):
  """Find a reasonably round epoch a small number above an existing one.

  Examples: 130-138     => 140
            139         => 150
            1234        => 1300
            19999-20998 => 21000
  """

  # Find the order of magnitude-1 (minimum: 1 as want no fractional values)
  small_magnitude = 10**max(1, int(floor(log(val, 10))-1))
  # How many units of this we have (float cast for __division__ insensitivity)
  mag_multiple = int(ceil(val / float(small_magnitude)))
  epoch = small_magnitude * mag_multiple
  # If this would give a consecutive number then offset it by a magnitude step
  if epoch <= val + 1:
    epoch = small_magnitude * (mag_multiple+1)
  return epoch

def _calculate_batch_offsets(experiments):
  """Take a list of experiments and resolve and return the batch offsets.

  This is the number added to the image number to give the
  batch number, such that:
  - Each experiment has a unique, nonoverlapping, nonconsecutive range
  - None are zero
  - Image number ranges are kept if at all possible
  """

  experiments_to_shift = []
  existing_ranges = set()
  maximum_batch_number = 0
  batch_offsets = [0]*len(experiments)

  # Handle zeroth shifts and kept ranges
  for i, experiment in enumerate(experiments):
    ilow, ihigh = experiment.image_range
    # Check assumptions
    assert ilow <= ihigh, "Inverted image order!?"
    assert ilow >= 0, "Negative image indices are not expected"
    # Don't emit zero: Causes problems with C/fortran number conversion
    if ilow == 0:
      ilow, ihigh = ilow+1, ihigh+1
    # If we overlap with anything, then process later
    if any( ilow <= high+1 and ihigh >= low-1 for low, high in existing_ranges):
      experiments_to_shift.append((i, experiment))
    else:
      batch_offsets[i] = ilow-experiment.image_range[0]
      existing_ranges.add((ilow, ihigh))
      maximum_batch_number = max(maximum_batch_number, ihigh)

  # Now handle all the experiments that overlapped by pushing them higher
  for i, experiment in experiments_to_shift:
    start_number = _next_epoch(maximum_batch_number)
    range_width = experiment.image_range[1]-experiment.image_range[0]+1
    end_number = start_number + range_width - 1
    batch_offsets[i] = start_number - experiment.image_range[0]
    maximum_batch_number = end_number
    experiment.scan.set_batch_offset(batch_offsets[i])

  return batch_offsets

def export_mtz(integrated_data, experiment_list, hklout,
               include_partials=False, keep_partials=False, scale_partials=True,
               min_isigi=None, force_static_model=False, filter_ice_rings=False,
               ignore_profile_fitting=False, apply_scales=False):
  '''Export data from integrated_data corresponding to experiment_list to an
  MTZ file hklout.'''

  # Convert experiment_list to a real python list or else identity assumptions
  # fail like:
  #   assert experiment_list[0] is experiment_list[0]
  # And assumptions about added attributes break
  experiment_list = list(experiment_list)

  if apply_scales:
    assert('inverse_scale_factor' in integrated_data)

  # Validate multi-experiment assumptions
  if len(experiment_list) > 1:
    # All experiments should match crystals, or else we need multiple crystals/datasets
    if not all(x.crystal == experiment_list[0].crystal for x in experiment_list[1:]):
      logger.warning("Warning: Experiment crystals differ. Using first experiment crystal for file-level data.")

    # We must match wavelengths (until multiple datasets supported)
    if not all(isclose(x.beam.get_wavelength(), experiment_list[0].beam.get_wavelength(), rel_tol=1e-9) for x in experiment_list[1:]):
      data = [x.beam.get_wavelength() for x in experiment_list]
      raise Sorry("Cannot export multiple experiments with different beam wavelengths ({})".format(data))

  # also only work correctly with one panel (for the moment)
  if any(len(experiment.detector) != 1 for experiment in experiment_list):
    logger.warning('Warning: Ignoring multiple panels in output MTZ')

  # Clean up the data with the passed in options
  integrated_data = _apply_data_filters(integrated_data,
      ignore_profile_fitting=ignore_profile_fitting,
      min_isigi=min_isigi,
      filter_ice_rings=filter_ice_rings,
      include_partials=include_partials,
      keep_partials=keep_partials,
      scale_partials=scale_partials)

  # Calculate and store the image range for each image
  for experiment in experiment_list:
    # Calculate this once so that we don't have to again
    if experiment.scan:
      experiment.image_range = experiment.scan.get_image_range()
    else:
      experiment.image_range = 1, 1

  batch_offsets = flex.int(
    expt.scan.get_batch_offset() for expt in experiment_list)
  if batch_offsets.all_eq(0):
    # Calculate any offset to the image numbers
    batch_offsets = _calculate_batch_offsets(experiment_list)
  else:
    unique_offsets = set(batch_offsets)
    if len(unique_offsets) != len(batch_offsets):
      import collections
      raise Sorry('Duplicate batch offsets detected: %s' %', '.join(
        str(item) for item, count in collections.Counter(batch_offsets).items()
        if count > 1))

  # Create the mtz file
  mtz_file = mtz.object()
  mtz_file.set_title('from dials.export_mtz')
  date_str = time.strftime('%d/%m/%Y at %H:%M:%S', time.gmtime())
  mtz_file.add_history('From %s, run on %s' % (dials_version(), date_str))

  # FIXME TODO for more than one experiment into an MTZ file:
  #
  # - add an epoch (or recover an epoch) from the scan and add this as an extra
  #   column to the MTZ file for scaling, so we know that the two lattices were
  #   integrated at the same time
  # ✓ decide a sensible BATCH increment to apply to the BATCH value between
  #   experiments and add this

  for experiment_index, experiment in enumerate(experiment_list):
    # Grab our subset of the data
    experiment.data = dict(integrated_data.select(integrated_data["id"] == experiment_index))

    # Do any crystal transformations for the experiment
    cb_op_to_ref = experiment.crystal.get_space_group().info(
        ).change_of_basis_op_to_reference_setting()
    experiment.crystal = experiment.crystal.change_basis(cb_op_to_ref)
    experiment.data["miller_index_rebase"] = cb_op_to_ref.apply(experiment.data["miller_index"])

    s0 = experiment.beam.get_s0()
    s0n = matrix.col(s0).normalize().elems
    logger.debug('Beam vector: %.4f %.4f %.4f' % s0n)

    for i in range(experiment.image_range[0], experiment.image_range[1]+1):
      _add_batch(mtz_file, experiment,
        batch_number=i+experiment.scan.get_batch_offset(),
        image_number=i,
        force_static_model=force_static_model)

    # Create the batch offset array. This gives us an experiment (id)-dependent
    # batch offset to calculate the correct batch from image number.
    experiment.data["batch_offset"] = flex.int(len(experiment.data["id"]), experiment.scan.get_batch_offset())

    # Calculate whether we have a ROT value for this experiment, and set the column
    _, _, z = experiment.data['xyzcal.px'].parts()
    if experiment.scan:
      experiment.data["ROT"] = experiment.scan.get_angle_from_array_index(z)
    else:
      experiment.data["ROT"] = z

  # Update the mtz general information now we've processed the experiments
  mtz_file.set_space_group_info(experiment_list[0].crystal.get_space_group().info())
  unit_cell = experiment_list[0].crystal.get_unit_cell()
  mtz_crystal = mtz_file.add_crystal('XTAL', 'DIALS', unit_cell.parameters())
  mtz_dataset = mtz_crystal.add_dataset('FROMDIALS', experiment_list[0].beam.get_wavelength())

  # Combine all of the experiment data columns before writing
  merged_data = {k: v.deep_copy() for k, v in experiment_list[0].data.items()}
  for experiment in experiment_list[1:]:
    for k, v in experiment.data.items():
      merged_data[k].extend(v)
  # ALL columns must be the same length
  assert len(set(len(v) for v in merged_data.values())) == 1, "Column length mismatch"
  assert len(merged_data["id"] == len(integrated_data["id"])), "Lost rows in split/combine"

  # Write all the data and columns to the mtz file
  _write_columns(mtz_file, mtz_dataset, merged_data,
    scale_partials=scale_partials, apply_scales=apply_scales)

  logger.info("Saving {} integrated reflections to {}".format(len(merged_data['id']), hklout))
  mtz_file.write(hklout)

  return mtz_file


# vim:set fileencoding=utf-8 ft=python ts=8 sw=4 sts=4 et cindent:
'''
Contains the methods and classes used for interaction with the database and for
storage of the repository's state.
'''
# Copyright © 2011  Fabian Knittel <fabian.knittel@lettink.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA.

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.schema import ForeignKey
from sqlalchemy.types import Integer
from sqlalchemy.types import String
from sqlalchemy.schema import Column
from sqlalchemy.schema import MetaData
from sqlalchemy.orm import relation
from sqlalchemy.orm import backref

_metadata = MetaData()
_Base = declarative_base(metadata=_metadata)

class Repository(_Base):
    """@DynamicAttrs"""
    __tablename__ = 'migrate_repositories'

    repository_id = Column(Integer, primary_key=True)
    repository_name = Column(String, unique=True)

    applied_patches = relation('AppliedPatch', backref=backref('repository',
            primaryjoin="Repository.repository_id == AppliedPatch.repository_id"),
            cascade='delete')

    def __init__(self, repository_id=None, repository_name=None):
        self.repository_id = repository_id
        self.repository_name = repository_name

    def __repr__(self):
        return "<MigrateRepository('%d','%s')>" % (self.repository_id,
                self.repository_name)

class AppliedPatch(_Base):
    """@DynamicAttrs"""
    __tablename__ = 'migrate_applied_patches'

    repository_id = Column(Integer,
            ForeignKey('migrate_repositories.repository_id'), primary_key=True)
    patch_name = Column(String, primary_key=True)

    def __init__(self, repository_id, patch_name):
        self.repository_id = repository_id
        self.patch_name = patch_name

    def __repr__(self):
        return "<AppliedPatch('%d','%s')>" % (self.repository_id,
                self.patch_name)

    @staticmethod
    def get_all(sess, repository_name):
        """Returns the list of applied patches for repository `repository_name`.
        """
        return sess.query(AppliedPatch)\
                .join(Repository)\
                .filter(Repository.repository_name == repository_name)\
                .all()

    @staticmethod
    def is_applied(sess, repository_name, patch_name):
        patch = sess.query(AppliedPatch)\
                .join(Repository)\
                .filter(Repository.repository_name == repository_name)\
                .filter(AppliedPatch.patch_name == patch_name)\
                .first()
        return patch is not None

DB_CLASSES = [Repository, AppliedPatch]

def create_tables(bind, checkfirst=True):
    dialect = bind.engine.dialect
    for dbcls in DB_CLASSES:
        dbcls.__table__.create(bind, checkfirst=checkfirst)
        if dialect.name == 'postgresql':
            bind.execute('GRANT SELECT ON %s TO PUBLIC' % dbcls.__table__.name)

def drop_tables(bind, checkfirst=True):
    for dbcls in reversed(DB_CLASSES):
        dbcls.__table__.drop(bind, checkfirst=checkfirst)

def _clear_connection(sess):
    """Patches might not always reset the connection's role or search_path, so
    explicitly do that here (currently only for PostgreSQL).
    """
    dialect = sess.connection().engine.dialect
    if dialect.name == 'postgresql':
        sess.execute('SET ROLE NONE')
        sess.execute('SET search_path = public')

def execute_script(sess, sql_text):
    """Execute an SQL script on a session. Works around limitation in SQLite
    back-end, which doesn't allow multiple statements, by using the
    SQLite-specific ``executescript()``-method, when available. The approach
    was borrowed from ``migrate.versioning.script.sql``.
    """
    dbapi = sess.connection().engine.raw_connection()
    if getattr(dbapi, 'executescript', None) is not None:
        dbapi.executescript(sql_text)
    else:
        sess.execute(sql_text)
    _clear_connection(sess)


#!/usr/bin/python3

import json
import os
import subprocess
import sys

from contextlib import contextmanager

@contextmanager
def change_directory(directory):
    original = os.path.abspath(os.getcwd())

    os.chdir(directory)
    yield
    os.chdir(original)

def build(directory, tags, options):
    with change_directory(directory):
        subprocess.run([
            'docker', 'build',
                *options,
                *[f'--tag={tag}' for tag in tags],
                '.'], check=True)

    for tag in tags:
        subprocess.run(['docker', 'push', tag], check=True)

if __name__ == '__main__':
    build_options = sys.argv[1:]

    with open('config.json', 'r', encoding='utf-8') as input_file:
        config = json.load(input_file)

    for item in config['builds']:
        build(item['dockerfile'], item['tags'], build_options)


nombres = ['Daniel', 'Jose', 'Miguel', 'Ana', 'Carolina']
# print(nombres)

# for element in nombres:
#     print(element)

listOfNumbers = [10, 50, -5, 20, -30.4]

# for number in listOfNumbers:
#     if number > 25:
#         print(number)

# print(listOfNumbers[1])

# for index, element in enumerate(listOfNumbers):
#     print(index, element)

matrix = [[1, 2 ,3], [4, 5, 6], [7, 8, 9]]

for row in matrix:
    print(row)
    for element in row:
        print(element)


#  ------------------------------------------------------------------------------------------
#  Copyright (c) Microsoft Corporation. All rights reserved.
#  Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
#  ------------------------------------------------------------------------------------------
from __future__ import annotations

import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence

import SimpleITK as sitk
import math
import numpy as np
import tensorboardX
import torch
import torch.nn.functional as F
from PIL import Image
from azureml.core import Run

from InnerEye.Azure.azure_util import DEFAULT_CROSS_VALIDATION_SPLIT_INDEX, PARENT_RUN_CONTEXT, RUN_CONTEXT, \
    get_run_context_or_default, is_offline_run_context
from InnerEye.Common.common_util import DataframeLogger
from InnerEye.Common.metrics_dict import MetricType, MetricsDict, ScalarMetricsDict, get_column_name_for_logging, \
    get_metric_name_with_hue_prefix
from InnerEye.Common.type_annotations import IntOrString, TupleFloat3
from InnerEye.ML.common import ModelExecutionMode
from InnerEye.ML.config import BACKGROUND_CLASS_NAME
from InnerEye.ML.model_config_base import ModelConfigBase
from InnerEye.ML.scalar_config import ScalarLoss
from InnerEye.ML.utils import metrics_util
from InnerEye.ML.utils.image_util import binaries_from_multi_label_array, check_array_range, is_binary_array
from InnerEye.ML.utils.io_util import reverse_tuple_float3
from InnerEye.ML.utils.metrics_constants import LoggingColumns
from InnerEye.ML.utils.metrics_util import binary_classification_accuracy, mean_absolute_error, r2_score
from InnerEye.ML.utils.ml_util import check_size_matches
from InnerEye.ML.utils.sequence_utils import get_masked_model_outputs_and_labels

TRAIN_STATS_FILE = "train_stats.csv"

@dataclass(frozen=True)
class InferenceMetrics:
    """
    Defined purely to serve as a superclass.
    """
    pass

@dataclass(frozen=True)
class InferenceMetricsForClassification(InferenceMetrics):
    """
    Stores a dictionary mapping from epoch number to the metrics that were achieved in that epoch.
    """
    epochs: Dict[int, MetricsDict]

@dataclass(frozen=True)
class InferenceMetricsForSegmentation(InferenceMetrics):
    """
    Stores metrics for segmentation models, per execution mode and epoch.
    """
    data_split: ModelExecutionMode
    epochs: Dict[int, float]

    def get_best_epoch_dice(self) -> float:
        """
        Gets the Dice score that the model achieves in the best epoch.
        """
        return self.epochs[self.get_best_epoch()]

    def get_best_epoch(self) -> int:
        """
        Gets the epoch that achieves the best (highest) Dice score.
        """
        epoch = max(self.epochs, key=self.epochs.get)
        return epoch

    def get_metrics_log_key(self) -> str:
        """
        Gets a string name for logging the metrics specific to the execution mode (train, val, test)
        :return:
        """
        return f"InferenceMetrics_{self.data_split.value}"

    def log_metrics(self, run_context: Run = None) -> None:
        """
        Log metrics for each epoch to the provided runs logs, or the current run context if None provided
        :param run_context: Run for which to log the metrics to, use the current run context if None provided
        :return:
        """
        run_context = get_run_context_or_default(run_context)
        keys = sorted(self.epochs.keys())

        run_context.log_table(name=self.get_metrics_log_key(), value={
            "Checkpoint": keys,
            "Dice": [self.epochs[x] for x in keys]
        })

@dataclass(frozen=True)
class SegmentationMetricsPerClass:
    """
    Stores different segmentation metrics, as a list where each list entry represents one class.
    """
    dice: List[float] = field(default_factory=list)
    hausdorff_distance_mm: List[float] = field(default_factory=list)
    mean_distance_mm: List[float] = field(default_factory=list)

    def append_nan(self) -> None:
        """
        Adds a NaN for all metrics that are stored, indicating that the target class was not present and no
        output was produced by the model.
        """
        self.dice.append(math.nan)
        self.hausdorff_distance_mm.append(math.nan)
        self.mean_distance_mm.append(math.nan)

    def append(self,
               dice: float,
               hausdorff_distance_mm: float,
               mean_distance_mm: float) -> None:
        """
        Stores the metrics for a class in the present object.
        :param dice: The Dice score between ground truth and model output.
        :param hausdorff_distance_mm: The Hausdorff distance between ground truth and model output, in millimeters.
        :param mean_distance_mm: The mean surface distance between ground truth and model output, in millimeters.
        :return:
        """
        self.dice.append(dice)
        self.hausdorff_distance_mm.append(hausdorff_distance_mm)
        self.mean_distance_mm.append(mean_distance_mm)

class AzureMLLogger:
    """
    Stores the information that is required to log metrics to AzureML.
    """

    def __init__(self,
                 cross_validation_split_index: int,
                 logging_prefix: str,
                 log_to_parent_run: bool):
        """
        :param cross_validation_split_index: The cross validation split index, or its default value if not running
        inside cross validation.
        :param logging_prefix: A prefix string that will be added to all metrics names before logging.
        :param log_to_parent_run: If true, all metrics will also be written to the Hyperdrive parent run when that
        parent run is present.
        """
        self.logging_prefix = logging_prefix
        self.cross_validation_split_index = cross_validation_split_index
        self.log_to_parent_run = log_to_parent_run

    def log_to_azure(self, label: str, metric: float) -> None:
        """
        Logs a metric as a key/value pair to AzureML.
        """
        if not is_offline_run_context(RUN_CONTEXT):
            metric_name = self.logging_prefix + label
            RUN_CONTEXT.log(metric_name, metric)
            # When running in a cross validation setting, log all metrics to the hyperdrive parent run too,
            # so that we can easily overlay graphs across runs.
            if self.log_to_parent_run and PARENT_RUN_CONTEXT:
                if self.cross_validation_split_index > DEFAULT_CROSS_VALIDATION_SPLIT_INDEX:
                    PARENT_RUN_CONTEXT.log(f"{metric_name}_Split{self.cross_validation_split_index}",
                                           metric)

class AzureAndTensorboardLogger:
    """
    Contains functionality to log metrics to both Azure run and TensorBoard event file
    for both classification and segmentation models.
    """

    def __init__(self,
                 azureml_logger: AzureMLLogger,
                 tensorboard_logger: tensorboardX.SummaryWriter,
                 epoch: int):
        self.azureml_logger = azureml_logger
        self.tensorboard_logger = tensorboard_logger
        self.epoch = epoch

    def log_to_azure_and_tensorboard(self, label: str, metric: float) -> None:
        """
        Writes a metric to the Azure run and to the TensorBoard event file
        :param label: The string name of the metric.
        :param metric: The value of the metric.
        """
        self.azureml_logger.log_to_azure(label, metric)
        self.log_to_tensorboard(label, metric)

    def log_to_tensorboard(self, label: str, metric: float) -> None:
        """
        Writes a metric to a Tensorboard event file.
        :param label: The string name of the metric.
        :param metric: The value of the metric.
        """
        # TensorBoard does not like tags that contain spaces, and prints out a warning for each logging attempt.
        # Replace space with underscore to reduce logging noise.
        writer = self.tensorboard_logger
        label_without_spaces = label.replace(" ", "_")
        writer.add_scalar(label_without_spaces, metric, self.epoch)

    def log_image(self, name: str, path: str) -> None:
        """
        Logs a PNG image stored in `path` to Azure and Tensorboard.
        """
        if not is_offline_run_context(RUN_CONTEXT):
            RUN_CONTEXT.log_image(name=name, path=path)
        writer = self.tensorboard_logger
        img = Image.open(path).convert("RGB")
        img = np.transpose(np.asarray(img), (2, 0, 1))
        writer.add_image(name, img, self.epoch)

    def log_segmentation_epoch_metrics(self,
                                       metrics: MetricsDict,
                                       learning_rates: List[float]) -> None:
        """
        Logs segmentation metrics (e.g. loss, dice scores, learning rates) to an event file for TensorBoard
        visualization and to the AzureML run context
        :param learning_rates: The logged learning rates.
        :param metrics: The metrics of the specified epoch, averaged along its batches.
        """
        logging_fn = self.log_to_azure_and_tensorboard
        logging_fn(MetricType.LOSS.value, metrics.get_single_metric(MetricType.LOSS))
        logging_fn("Dice/AverageExceptBackground", metrics.get_single_metric(MetricType.DICE))
        logging_fn("Voxels/ProportionForeground", metrics.get_single_metric(MetricType.PROPORTION_FOREGROUND_VOXELS))
        logging_fn("TimePerEpoch_Seconds", metrics.get_single_metric(MetricType.SECONDS_PER_EPOCH))

        if learning_rates is not None:
            for i, lr in enumerate(learning_rates):
                logging_fn("LearningRate/Index_{}".format(i), lr)

        for class_name in metrics.get_hue_names(include_default=False):
            # Tensorboard groups metrics by what is before the slash.
            # With metrics Dice/Foo and Dice/Bar, it will create a section for "Dice",
            # and inside of it, there are graphs for Foo and Bar
            get_label = lambda x, y: "{}/{}".format(x, y)
            logging_fn(get_label("Dice", class_name),
                       metrics.get_single_metric(MetricType.DICE, hue=class_name))
            logging_fn(get_label("Voxels", class_name),
                       metrics.get_single_metric(MetricType.PROPORTION_FOREGROUND_VOXELS, hue=class_name))

    def log_classification_epoch_metrics(self, metrics: MetricsDict) -> None:
        """
        Writes all values from MetricsDict object into a file for Tensorboard visualization,
        and into the AzureML run context.
        :param metrics: dictionary containing the metrics to be logged, averaged over minibatches.
        """
        for hue_name, label, metric in metrics.enumerate_single_values():
            self.log_to_azure_and_tensorboard(get_metric_name_with_hue_prefix(label, hue_name), metric)

def vars_with_scalar_fields_only(o: Any) -> Dict[str, Any]:
    """
    Returns a dictionary similar to vars(o), but with only those fields that either have integer
    or floating point value.
    :param o: The object to process.
    :return: A dictionary mapping from field name to value.
    """

    def is_scalar(f: Any) -> bool:
        return isinstance(f, (int, float))

    return {key: value for key, value in vars(o) if is_scalar(value)}

def surface_distance(seg: sitk.Image, reference_segmentation: sitk.Image) -> float:
    """
    Symmetric surface distances taking into account the image spacing
    https://github.com/InsightSoftwareConsortium/SimpleITK-Notebooks/blob/master/Python/34_Segmentation_Evaluation.ipynb
    :param seg: mask 1
    :param reference_segmentation: mask 2
    :return: mean distance
    """
    statistics_image_filter = sitk.StatisticsImageFilter()
    # Get the number of pixels in the reference surface by counting all pixels that are 1.
    reference_surface = sitk.LabelContour(reference_segmentation)
    statistics_image_filter.Execute(reference_surface)
    num_reference_surface_pixels = int(statistics_image_filter.GetSum())

    reference_distance_map = sitk.Abs(
        sitk.SignedMaurerDistanceMap(reference_segmentation, squaredDistance=False, useImageSpacing=True))
    reference_surface = sitk.LabelContour(reference_segmentation)

    # Symmetric surface distance measures
    segmented_distance_map = sitk.Abs(sitk.SignedMaurerDistanceMap(seg, squaredDistance=False, useImageSpacing=True))
    segmented_surface = sitk.LabelContour(seg)

    # Multiply the binary surface segmentations with the distance maps. The resulting distance
    # maps contain non-zero values only on the surface (they can also contain zero on the surface)
    seg2ref_distance_map = reference_distance_map * sitk.Cast(segmented_surface, sitk.sitkFloat32)
    ref2seg_distance_map = segmented_distance_map * sitk.Cast(reference_surface, sitk.sitkFloat32)

    # Get the number of pixels in the reference surface by counting all pixels that are 1.
    statistics_image_filter.Execute(segmented_surface)
    num_segmented_surface_pixels = int(statistics_image_filter.GetSum())

    seg2ref_distance_map_arr = sitk.GetArrayViewFromImage(seg2ref_distance_map)
    seg2ref_distances = _add_zero_distances(num_segmented_surface_pixels, seg2ref_distance_map_arr)
    ref2seg_distance_map_arr = sitk.GetArrayViewFromImage(ref2seg_distance_map)
    ref2seg_distances = _add_zero_distances(num_reference_surface_pixels, ref2seg_distance_map_arr)

    all_surface_distances = seg2ref_distances + ref2seg_distances
    return np.mean(all_surface_distances).item()

def _add_zero_distances(num_segmented_surface_pixels: int, seg2ref_distance_map_arr: np.ndarray) -> List[float]:
    """
    # Get all non-zero distances and then add zero distances if required.
    :param num_segmented_surface_pixels:
    :param seg2ref_distance_map_arr:
    :return: list of distances, augmented with zeros.
    """
    seg2ref_distances = list(seg2ref_distance_map_arr[seg2ref_distance_map_arr != 0])
    seg2ref_distances = seg2ref_distances + list(np.zeros(num_segmented_surface_pixels - len(seg2ref_distances)))
    return seg2ref_distances

def calculate_metrics_per_class(segmentation: np.ndarray,
                                ground_truth: np.ndarray,
                                ground_truth_ids: List[str],
                                voxel_spacing: TupleFloat3,
                                patient_id: Optional[int] = None) -> MetricsDict:
    """
    Calculate the dice for all foreground structures (the background class is completely ignored).
    Returns a MetricsDict with metrics for each of the foreground
    structures. Metrics are NaN if both ground truth and prediction are all zero for a class.
    :param ground_truth_ids: The names of all foreground classes.
    :param segmentation: predictions multi-value array with dimensions: [Z x Y x X]
    :param ground_truth: ground truth binary array with dimensions: [C x Z x Y x X]
    :param voxel_spacing: voxel_spacing in 3D Z x Y x X
    :param patient_id: for logging
    """
    number_of_classes = ground_truth.shape[0]
    if len(ground_truth_ids) != (number_of_classes - 1):
        raise ValueError(f"Received {len(ground_truth_ids)} foreground class names, but "
                         f"the label tensor indicates that there are {number_of_classes - 1} classes.")
    binaries = binaries_from_multi_label_array(segmentation, number_of_classes)

    all_classes_are_binary = [is_binary_array(ground_truth[label_id]) for label_id in range(ground_truth.shape[0])]
    if not np.all(all_classes_are_binary):
        raise ValueError("Ground truth values should be 0 or 1")
    overlap_measures_filter = sitk.LabelOverlapMeasuresImageFilter()
    hausdorff_distance_filter = sitk.HausdorffDistanceImageFilter()
    metrics = MetricsDict(hues=ground_truth_ids)
    for i, prediction in enumerate(binaries):
        if i == 0:
            continue
        check_size_matches(prediction, ground_truth[i], arg1_name="prediction", arg2_name="ground_truth")
        if not is_binary_array(prediction):
            raise ValueError("Predictions values should be 0 or 1")
        # simpleitk returns a Dice score of 0 if both ground truth and prediction are all zeros.
        # We want to be able to fish out those cases, and treat them specially later.
        prediction_zero = np.all(prediction == 0)
        gt_zero = np.all(ground_truth[i] == 0)
        dice = mean_surface_distance = hausdorff_distance = math.nan
        if not (prediction_zero and gt_zero):
            prediction_image = sitk.GetImageFromArray(prediction.astype(np.uint8))
            prediction_image.SetSpacing(sitk.VectorDouble(reverse_tuple_float3(voxel_spacing)))
            ground_truth_image = sitk.GetImageFromArray(ground_truth[i].astype(np.uint8))
            ground_truth_image.SetSpacing(sitk.VectorDouble(reverse_tuple_float3(voxel_spacing)))
            overlap_measures_filter.Execute(prediction_image, ground_truth_image)
            dice = overlap_measures_filter.GetDiceCoefficient()
            if prediction_zero or gt_zero:
                hausdorff_distance = mean_surface_distance = math.inf
            else:
                try:
                    hausdorff_distance_filter.Execute(prediction_image, ground_truth_image)
                    hausdorff_distance = hausdorff_distance_filter.GetHausdorffDistance()
                except Exception as e:
                    logging.warning("Cannot calculate Hausdorff distance for "
                                    f"structure {i} of patient {patient_id}: {e}")
                try:
                    mean_surface_distance = surface_distance(prediction_image, ground_truth_image)
                except Exception as e:
                    logging.warning(f"Cannot calculate mean distance for structure {i} of patient {patient_id}: {e}")
            logging.debug(f"Patient {patient_id}, class {i} has Dice score {dice}")

        def add_metric(metric_type: MetricType, value: float) -> None:
            metrics.add_metric(metric_type, value, skip_nan_when_averaging=True, hue=ground_truth_ids[i - 1])

        add_metric(MetricType.DICE, dice)
        add_metric(MetricType.HAUSDORFF_mm, hausdorff_distance)
        add_metric(MetricType.MEAN_SURFACE_DIST_mm, mean_surface_distance)
    return metrics

def compute_dice_across_patches(segmentation: torch.Tensor,
                                ground_truth: torch.Tensor,
                                use_cuda: bool,
                                allow_multiple_classes_for_each_pixel: bool = False) -> torch.Tensor:
    """
    Computes the Dice scores for all classes across all patches in the arguments.
    :param segmentation: Tensor containing class ids predicted by a model.
    :param ground_truth: One-hot encoded torch tensor containing ground-truth label ids.
    :param use_cuda: If set to True, uses CUDA backend for computations
    :param allow_multiple_classes_for_each_pixel: If set to False, ground-truth tensor has
    to contain only one foreground label for each pixel.
    :return A torch tensor of size (Patches, Classes) with the Dice scores. Dice scores are computed for
    all classes including the background class at index 0.
    """
    if use_cuda:
        segmentation = segmentation.cuda()
        ground_truth = ground_truth.cuda()

    check_size_matches(segmentation, ground_truth, 4, 5, [0, -3, -2, -1],
                       arg1_name="segmentation", arg2_name="ground_truth")

    # One-hot encoded ground-truth values should sum up to one for all pixels
    if not allow_multiple_classes_for_each_pixel:
        if not torch.allclose(torch.sum(ground_truth, dim=1).float(),
                              torch.ones(segmentation.shape, device=ground_truth.device).float()):
            raise Exception("Ground-truth one-hot matrix does not sum up to one for all pixels")

    # Convert the ground-truth to one-hot-encoding
    [num_patches, num_classes] = ground_truth.size()[:2]
    one_hot_segmentation = F.one_hot(segmentation, num_classes=num_classes).permute(0, 4, 1, 2, 3)

    # Convert the tensors to bool tensors
    one_hot_segmentation = one_hot_segmentation.bool().view(num_patches, num_classes, -1)
    ground_truth = ground_truth.bool().view(num_patches, num_classes, -1)

    # And operation between segmentation and ground-truth - reduction operation
    # Count the number of samples in segmentation and ground-truth
    intersection = 2.0 * torch.sum(one_hot_segmentation & ground_truth, dim=-1).float()
    union = torch.sum(one_hot_segmentation, dim=-1) + torch.sum(ground_truth, dim=-1).float() + 1.0e-6

    return intersection / union

def format_learning_rates(learning_rates: List[float]) -> str:
    """
    Converts a list of learning rates to a human readable string. Multiple entries are separated by semicolon.
    :param learning_rates: An iterable of learning rate values.
    :return: An empty string if the argument is None or empty, otherwise the string representation of the rates,
    formatted as {:0.2e}
    """
    if learning_rates is None or len(learning_rates) == 0:
        return ""
    return "; ".join("{:0.2e}".format(lr) for lr in learning_rates)

def store_epoch_stats_for_segmentation(outputs_dir: Path,
                                       epoch: int,
                                       learning_rates: List[float],
                                       training_results: MetricsDict,
                                       validation_results: MetricsDict) -> None:
    """
    Writes a dictionary of statistics for a segmentation training run to a file. Successive calls to the function
    append another line of metrics. The first line of the file contains the column headers (names of the metrics).
    :param training_results: A MetricsDict object with all metrics that were achieved on the training set in the
    current epoch.
    :param validation_results: A MetricsDict object with all metrics that were achieved on the validation set in the
    current epoch.
    :param learning_rates: The learning rates that were used in the current epoch.
    :param epoch: The number of the current training epoch.
    :param outputs_dir: The directory in which the statistics file should be created.
    :return:
    """
    epoch_stats = {
        "Epoch": str(epoch),
        "LearningRate": format_learning_rates(learning_rates),
        "TrainLoss": metrics_util.format_metric(training_results.get_single_metric(MetricType.LOSS)),
        "TrainDice": metrics_util.format_metric(training_results.get_single_metric(MetricType.DICE)),
        "ValLoss": metrics_util.format_metric(validation_results.get_single_metric(MetricType.LOSS)),
        "ValDice": metrics_util.format_metric(validation_results.get_single_metric(MetricType.DICE)),
    }
    # When using os.linesep, additional LF characters are inserted. Expected behaviour only when
    # using this on both Windows and Linux.
    line_sep = "\n"
    tab = "\t"
    full_file = outputs_dir / TRAIN_STATS_FILE
    if not full_file.exists():
        header = tab.join(epoch_stats.keys())
        full_file.write_text(header + line_sep)
    line = tab.join(epoch_stats.values())
    with full_file.open("a") as f:
        f.write(line + line_sep)

def validate_and_store_model_parameters(writer: tensorboardX.SummaryWriter, epoch: int,
                                        model: torch.nn.DataParallel) -> None:
    """
    Validates and writes all model weights to the given TensorBoard writer.
    :param writer: TensorBoard summary writer
    :param epoch: The epoch for which these model parameters correspond to.
    :param model: The model from which to extract the parameters.
    :return:
    """
    for name, param in model.named_parameters():
        param_numpy = param.clone().cpu().data.numpy()
        check_array_range(param_numpy, error_prefix="Parameter {}".format(name))
        writer.add_histogram(name, param_numpy, epoch)

def store_epoch_metrics(azure_and_tensorboard_logger: AzureAndTensorboardLogger,
                        df_logger: DataframeLogger,
                        epoch: int,
                        metrics: MetricsDict,
                        learning_rates: List[float],
                        config: ModelConfigBase) -> None:
    """
    Writes the loss, Dice scores, and learning rates into a file for Tensorboard visualization,
    and into the AzureML run context.
    :param azure_and_tensorboard_logger: An instance of AzureAndTensorboardLogger.
    :param df_logger: An instance of DataframeLogger, for logging results to csv.
    :param epoch: The epoch corresponding to the results.
    :param metrics: The metrics of the specified epoch, averaged along its batches.
    :param learning_rates: The logged learning rates.
    :param config: one of SegmentationModelBase
    """
    if config.is_segmentation_model:
        azure_and_tensorboard_logger.log_segmentation_epoch_metrics(metrics,
                                                                    learning_rates)
        logger_row = {
            LoggingColumns.Dice.value: metrics.get_single_metric(MetricType.DICE),
            LoggingColumns.Loss.value: metrics.get_single_metric(MetricType.LOSS),
            LoggingColumns.SecondsPerEpoch.value: metrics.get_single_metric(MetricType.SECONDS_PER_EPOCH)
        }

    elif config.is_scalar_model:
        assert isinstance(metrics, MetricsDict)
        azure_and_tensorboard_logger.log_classification_epoch_metrics(metrics)
        logger_row: Dict[str, float] = {}  # type: ignore
        for hue_name, metric_name, metric_value in metrics.enumerate_single_values():
            logging_column_name = get_column_name_for_logging(metric_name, hue_name=hue_name)
            logger_row[logging_column_name] = metric_value
    else:
        raise ValueError("Model must be either classification, regression or segmentation model")

    logger_row.update({
        LoggingColumns.Epoch.value: epoch,
        LoggingColumns.CrossValidationSplitIndex.value: config.cross_validation_split_index
    })

    df_logger.add_record(logger_row)

def compute_scalar_metrics(metrics_dict: ScalarMetricsDict,
                           subject_ids: Sequence[IntOrString],
                           model_output: torch.Tensor,
                           labels: torch.Tensor,
                           loss_type: ScalarLoss = ScalarLoss.BinaryCrossEntropyWithLogits) -> None:
    """
    Computes various metrics for a binary classification task from real-valued model output and a label vector,
    and stores them in the given `metrics_dict`.
    The model output is assumed to be in the range between 0 and 1, a value larger than 0.5 indicates a prediction
    of class 1. The label vector is expected to contain class indices 0 and 1 only.
    Metrics for each model output channel will be isolated, and a non-default hue for each model output channel is
    expected, and must exist in the provided metrics_dict. The Default hue is used for single model outputs.
    :param metrics_dict: An object that holds all metrics. It will be updated in-place.
    :param subject_ids: Subject ids for the model output and labels.
    :param model_output: A tensor containing model outputs.
    :param labels: A tensor containing class labels.
    :param loss_type: The type of loss that the model uses. This is required to optionally convert 2-dim model output
    to probabilities.
    """
    _model_output_channels = model_output.shape[1]
    model_output_hues = metrics_dict.get_hue_names(include_default=len(metrics_dict.hues_without_default) == 0)

    if len(model_output_hues) < _model_output_channels:
        raise ValueError("Hues must be provided for each model output channel, found "
                         f"{_model_output_channels} channels but only {len(model_output_hues)} hues")

    for i, hue in enumerate(model_output_hues):
        # mask the model outputs and labels if required
        masked_model_outputs_and_labels = get_masked_model_outputs_and_labels(
            model_output[:, i, ...], labels[:, i, ...], subject_ids)

        # compute metrics on valid masked tensors only
        if masked_model_outputs_and_labels is not None:
            _model_output, _labels, _subject_ids = \
                masked_model_outputs_and_labels.model_outputs.data, \
                masked_model_outputs_and_labels.labels.data, \
                masked_model_outputs_and_labels.subject_ids

            if loss_type == ScalarLoss.MeanSquaredError:
                metrics = {
                    MetricType.MEAN_SQUARED_ERROR: F.mse_loss(_model_output, _labels.float(), reduction='mean').item(),
                    MetricType.MEAN_ABSOLUTE_ERROR: mean_absolute_error(_model_output, _labels),
                    MetricType.R2_SCORE: r2_score(_model_output, _labels)
                }
            else:
                metrics = {
                    MetricType.CROSS_ENTROPY: F.binary_cross_entropy(_model_output, _labels.float(),
                                                                     reduction='mean').item(),
                    MetricType.ACCURACY_AT_THRESHOLD_05: binary_classification_accuracy(_model_output, _labels)
                }
            for key, value in metrics.items():
                if key == MetricType.R2_SCORE:
                    # For a batch size 1, R2 score can be nan. We need to ignore nans
                    # when average in case the last batch is of size 1.
                    metrics_dict.add_metric(key, value, skip_nan_when_averaging=True, hue=hue)
                else:
                    metrics_dict.add_metric(key, value, hue=hue)

            assert _subject_ids is not None
            metrics_dict.add_predictions(_subject_ids, _model_output.detach().cpu().numpy(),
                                         _labels.cpu().numpy(), hue=hue)

def aggregate_segmentation_metrics(metrics: MetricsDict) -> MetricsDict:
    """
    Computes aggregate metrics for segmentation models, from a metrics dictionary that contains the results for
    individual minibatches. Specifically, average Dice scores for only the foreground structures and proportions
    of foreground voxels are computed. All metrics for the background class will be removed.
    All other metrics that are already present in the input metrics will be averaged and available in the result.
    Diagnostic values present in the input will be passed through unchanged.
    :param metrics: A metrics dictionary that contains the per-minibatch results.
    """
    class_names_with_background = metrics.get_hue_names(include_default=False)
    has_background_class = class_names_with_background[0] == BACKGROUND_CLASS_NAME
    foreground_classes = class_names_with_background[1:] if has_background_class else class_names_with_background
    result = metrics.average(across_hues=False)
    result.diagnostics = metrics.diagnostics.copy()
    if has_background_class:
        result.delete_hue(BACKGROUND_CLASS_NAME)
    add_average_foreground_dice(result)
    # Total number of voxels per class, including the background class
    total_voxels = []
    voxel_count = MetricType.VOXEL_COUNT.value
    for g in class_names_with_background:
        values = metrics.values(hue=g)
        if voxel_count in values:
            total_voxels.append(sum(values[voxel_count]))
    if len(total_voxels) > 0:
        # Proportion of voxels in foreground classes only
        proportion_foreground = np.array(total_voxels[1:], dtype=float) / sum(total_voxels)
        for i, foreground_class in enumerate(foreground_classes):
            result.add_metric(MetricType.PROPORTION_FOREGROUND_VOXELS, proportion_foreground[i], hue=foreground_class)
        result.add_metric(MetricType.PROPORTION_FOREGROUND_VOXELS, np.sum(proportion_foreground).item())
    return result

def add_average_foreground_dice(metrics: MetricsDict) -> None:
    """
    If the given metrics dictionary contains an entry for Dice score, and only one value for the Dice score per class,
    then add an average Dice score for all foreground classes to the metrics dictionary (modified in place).
    :param metrics: The object that holds metrics. The average Dice score will be written back into this object.
    """
    all_dice = []
    for structure_name in metrics.get_hue_names(include_default=False):
        if structure_name != BACKGROUND_CLASS_NAME:
            all_dice.append(metrics.get_single_metric(MetricType.DICE, hue=structure_name))
    metrics.add_metric(MetricType.DICE, np.nanmean(all_dice).item())


class Solution:
    # @param a : list of integers
    # @param b : integer
    # @return a list of integers
    # rotate the array A by B positions.
    # A : [1 2 3 4 5 6]
	# B : 1
	# output : [2 3 4 5 6 1]
    def rotateArray(self, a, b):
        ret = []

        for i in xrange(len(a)):
            newIndex = (i+b) % len(a)
            ret.append(a[newIndex])

        return ret


import json
import uuid
import urllib
import urllib2

def search(name):
    id = str(uuid.uuid4())
    url =  'https://preview.academic.microsoft.com/api/search/GetEntityResults?correlationId=' + id

    query = {'Query':'@' + name +  '@'}
    headers = { 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36',
                'Accept':'*/*',
                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8' ,
                'Origin': 'https://preview.academic.microsoft.com',
                'Referer' : 'https://preview.academic.microsoft.com/',
                'Accept':'application/json',
    }
    form = urllib.urlencode(query) + '&Limit=1&Offset=0&OrderBy=&SortAscending=false'
    request = urllib2.Request(url, form, headers)
    results = urllib2.urlopen(request)
#     resp = requests.post(url, data=payload, headers=headers)

#     pretty_print_POST(resp.request)
    data = json.loads(results.read())
    file1 = open(name + "reponse1.json","w")
    file1.write(json.dumps(data, indent=4, sort_keys=False))
    file1.close()

    article = data['entitiesInQuery'][0]

    filterAuthors = []
    for autor in article['entity']['aa']:
        filterAuthors.append('Composite(AA.AuId%3D' + str(autor['auId']) + ')%2C')

    filterAuthors = '&Filters=' + ''.join(filterAuthors).rsplit('%2C', 1)[0]

    form2 = urllib.urlencode(query)  + filterAuthors +'&Limit=1&Offset=0&OrderBy=&SortAscending=false'
    request = urllib2.Request(url, form2, headers)
    results = urllib2.urlopen(request)

    data = json.loads(results.read())

    file3 = open(name + "reponse2.json","w")
    file3.write(json.dumps(data, indent=4, sort_keys=False))
    file3.close()

def main():
    search('Accuracy and Diversity in Cross-domain Recommendations for Cold-start Users with Positive-only Feedback')

def pretty_print_POST(req):
    """
    At this point it is completely built and ready
    to be fired; it is "prepared".

    However pay attention at the formatting used in
    this function because it is programmed to be pretty
    printed and may differ from the actual request.
    """
    print('{}\n{}\n{}\n\n{}'.format(
        '-----------START-----------',
        req.method + ' ' + req.url,
        '\n'.join('{}: {}'.format(k, v) for k, v in req.headers.items()),
        req.body,
    ))

if __name__ == '__main__':
    main()


try:
    import os, sys, json
    import numpy as np
    import torch
    import torch.nn as nn
    import math
    from ..._utils.pascal_voc_rectangles import _reconstruct
    from .util import normalize_batch
    HAS_TORCH = True
except Exception as e:
    HAS_TORCH = False

try:
    import arcpy
except:
    pass

def pred2dict(bb_np, score, cat_str, c):
    '''
    Create a dictionary with the attributes of a single predicted bounding box

    Parameters
    bb_np: bounding box coordinates (ymin, xmin, ymax, xmax)
    score: prediction probability
    category: name of the predicted class
    class: index of the the predicted class

    returns: a dictionarty with bounding box attributes
    '''
    # convert to top left x,y bottom right x,y
    return {"x1": bb_np[1],
            "x2": bb_np[3],
            "y1": bb_np[0],
            "y2": bb_np[2],
            "score": score,
            "category": cat_str,
            "class": c}

def convert_bounding_boxes_to_coord_list(bounding_boxes):
    '''
    Convert bounding box numpy array to python list of point arrays.
    The points will represent the corners of a polygon.

    Parameters
    bounding_boxes: numpy array of shape [n, 4]

    return: python array of point numpy arrays, each point array is in shape [4,2]
            representing coordinates (y,x) of the polygon points starting from top-left corner
    '''
    num_bounding_boxes = bounding_boxes.shape[0]
    bounding_box_coord_list = []
    for i in range(num_bounding_boxes):
        coord_array = np.empty(shape=(4, 2), dtype=np.float)
        coord_array[0][0] = bounding_boxes[i][0]
        coord_array[0][1] = bounding_boxes[i][1]

        coord_array[1][0] = bounding_boxes[i][0]
        coord_array[1][1] = bounding_boxes[i][3]

        coord_array[2][0] = bounding_boxes[i][2]
        coord_array[2][1] = bounding_boxes[i][3]

        coord_array[3][0] = bounding_boxes[i][2]
        coord_array[3][1] = bounding_boxes[i][1]

        bounding_box_coord_list.append(coord_array)

    return bounding_box_coord_list

def calculate_rectangle_size_from_batch_size(batch_size):
    '''
    Calculate number of rows and cols of image chips to composite a rectanglular block given a batch size

    Parameters
    batch_size: number of image chips in a batch

    return: number of cols and rows of image chips
    '''
    rectangle_height = int(math.sqrt(batch_size) + 0.5)
    rectangle_width = int(batch_size / rectangle_height)

    if rectangle_height * rectangle_width > batch_size:
        if rectangle_height >= rectangle_width:
            rectangle_height = rectangle_height - 1
        else:
            rectangle_width = rectangle_width - 1

    if (rectangle_height + 1) * rectangle_width <= batch_size:
        rectangle_height = rectangle_height + 1
    if (rectangle_width + 1) * rectangle_height <= batch_size:
        rectangle_width = rectangle_width + 1

    # swap col and row to make a horizontal rect
    if rectangle_height > rectangle_width:
        rectangle_height, rectangle_width = rectangle_width, rectangle_height

    if rectangle_height * rectangle_width != batch_size:
        return batch_size, 1

    return rectangle_height, rectangle_width

def get_tile_size(model_height, model_width, padding, batch_height, batch_width):
    '''
    Calculate request tile size given model and batch dimensions
    :param model_height:
    :param model_width:
    :param padding:
    :param batch_width:
    :param batch_height:
    :return: tile height and tile width
    '''
    tile_height = (model_height - 2 * padding) * batch_height
    tile_width = (model_width - 2 * padding) * batch_width

    return tile_height, tile_width

def tile_to_batch(pixel_block, model_height, model_width, padding, fixed_tile_size=True, **kwargs):
    inner_width = model_width - 2 * padding
    inner_height = model_height - 2 * padding

    band_count, pb_height, pb_width = pixel_block.shape
    pixel_type = pixel_block.dtype

    if fixed_tile_size is True:
        batch_height = kwargs['batch_height']
        batch_width = kwargs['batch_width']
    else:
        batch_height = math.ceil((pb_height - 2 * padding) / inner_height)
        batch_width = math.ceil((pb_width - 2 * padding) / inner_width)

    batch = np.zeros(shape=(batch_width * batch_height, band_count, model_height, model_width), dtype=pixel_type)
    for b in range(batch_width * batch_height):
        y = int(b / batch_width)
        x = int(b % batch_width)

        # pixel block might not be the shape (band_count, model_height, model_width)
        sub_pixel_block = pixel_block[:, y * inner_height: y * inner_height + model_height,
                    x * inner_width: x * inner_width + model_width]
        sub_pixel_block_shape = sub_pixel_block.shape
        batch[b, :, :sub_pixel_block_shape[1], :sub_pixel_block_shape[2]] = sub_pixel_block

    return batch, batch_height, batch_width

def batch_to_tile(batch, batch_height, batch_width):
    batch_size, bands, inner_height, inner_width = batch.shape
    tile = np.zeros(shape=(bands, inner_height * batch_height, inner_width * batch_width), dtype=batch.dtype)

    for b in range(batch_width * batch_height):
        y = int(b / batch_width)
        x = int(b % batch_width)

        tile[:, y * inner_height: (y+1) * inner_height, x * inner_width:(x+1) * inner_width] = batch[b]

    return tile

def remove_bounding_boxes_in_padding(bounding_boxes, scores, classes, image_height, image_width, padding,
                                     batch_height=1, batch_width=1):
    '''

    :param bounding_boxes: the batch of bounding boxes, shape=[B,N,4]
    :param scores: the batch of box scores, shape=[B,N]
    :param classes: the batch of labels, shape=[B,N]
    :param image_height: model height
    :param image_width: model width
    :param padding:
    :param batch_height:
    :param batch_width:
    :return:
    '''
    keep_indices = np.where((bounding_boxes[:,:,0] < image_height-padding) &
              (bounding_boxes[:,:,1] < image_width-padding) &
              (bounding_boxes[:,:,2] > padding) &
              (bounding_boxes[:,:,3] > padding))

    inner_width = image_width - 2 * padding
    inner_height = image_height - 2 * padding

    # convert coordinates in the batch to super tile and then filter by the keep_indices
    for b in range(batch_width * batch_height):
        y = int(b / batch_width)
        x = int(b % batch_width)

        bounding_boxes[b, :, [0, 2]] = bounding_boxes[b, :, [0, 2]] + y * inner_height
        bounding_boxes[b, :, [1, 3]] = bounding_boxes[b, :, [1, 3]] + x * inner_width

    bounding_boxes = bounding_boxes[keep_indices]
    scores = scores[keep_indices]
    classes = classes[keep_indices]

    return bounding_boxes, scores, classes

class ChildObjectDetector:

    def initialize(self, model, model_as_file):

        if not HAS_TORCH:
            raise Exception('PyTorch is not installed. Install it using conda install -c pytorch pytorch torchvision')

        import arcgis
        from arcgis.learn.models import YOLOv3

        if arcpy.env.processorType == "GPU" and torch.cuda.is_available():
            self.device = torch.device('cuda')
            arcgis.env._processorType = "GPU"
        else:
            self.device = torch.device('cpu')
            arcgis.env._processorType = "CPU"

        if model_as_file:
            with open(model, 'r') as f:
                self.json_info = json.load(f)
        else:
            self.json_info = json.load(model)

        model_path = self.json_info['ModelFile']
        if model_as_file and not os.path.isabs(model_path):
            model_path = os.path.abspath(os.path.join(os.path.dirname(model), model_path))

        self.model = YOLOv3.from_model(emd_path=model)
        self.model.learn.model = self.model.learn.model.to(self.device)
        self.model.learn.model.eval()

    def getParameterInfo(self, required_parameters):
        required_parameters.extend(
            [
                {
                    'name': 'padding',
                    'dataType': 'numeric',
                    'value': self.json_info['ImageHeight'] // 4,
                    'required': False,
                    'displayName': 'Padding',
                    'description': 'Padding'
                },
                {
                    'name': 'threshold',
                    'dataType': 'numeric',
                    'value': 0.1,
                    'required': False,
                    'displayName': 'Confidence Score Threshold [0.0, 1.0]',
                    'description': 'Confidence score threshold value [0.0, 1.0]'
                },
                {
                    'name': 'nms_overlap',
                    'dataType': 'numeric',
                    'value': 0.1,
                    'required': False,
                    'displayName': 'NMS Overlap',
                    'description': 'Maximum allowed overlap within each chip'
                },
                {
                    'name': 'batch_size',
                    'dataType': 'numeric',
                    'required': False,
                    'value': 4,
                    'displayName': 'Batch Size',
                    'description': 'Batch Size'
                },
                {
                    'name': 'exclude_pad_detections',
                    'dataType': 'string',
                    'required': False,
                    'domain': ('True', 'False'),
                    'value': 'True',
                    'displayName': 'Filter Outer Padding Detections',
                    'description': 'Filter detections which are outside the specified padding'
                }

            ]
        )
        return required_parameters

    def getConfiguration(self, **scalars):
        self.padding = int(scalars.get('padding', self.json_info['ImageHeight'] // 4)) ## Default padding Imageheight//4.
        self.nms_overlap = float(scalars.get('nms_overlap', 0.1))  ## Default 0.1 NMS Overlap.
        self.thres = float(scalars.get('threshold', 0.1)) ## Default 0.1 threshold.
        self.batch_size = int(math.sqrt(int(scalars.get('batch_size', 4)))) ** 2  ## Default 4 batch_size
        self.filter_outer_padding_detections = scalars.get('exclude_pad_detections', 'True').lower() in ['true', '1', 't', 'y', 'yes'] ## Default value True

        self.rectangle_height, self.rectangle_width = calculate_rectangle_size_from_batch_size(self.batch_size)
        ty, tx = get_tile_size(self.json_info['ImageHeight'], self.json_info['ImageWidth'],
                                         self.padding, self.rectangle_height, self.rectangle_width)

        return {
            'extractBands': tuple(self.json_info['ExtractBands']),
            'padding': self.padding,
            'threshold': self.thres,
            'nms_overlap': self.nms_overlap,
            'tx': tx,
            'ty': ty,
            'fixedTileSize': 1
        }

    def vectorize(self, **pixelBlocks):
        input_image = pixelBlocks['raster_pixels']
        batch, batch_height, batch_width = \
            tile_to_batch(input_image,
                                    self.json_info['ImageHeight'],
                                    self.json_info['ImageWidth'],
                                    self.padding,
                                    fixed_tile_size=True,
                                    batch_height=self.rectangle_height,
                                    batch_width=self.rectangle_width)

        class_names = [clas['Name'] for clas in self.json_info['Classes']]

        class dummy():
            pass

        dummy_x = dummy()
        chip_size = self.json_info["ImageHeight"]
        dummy_x.size = [chip_size, chip_size]

        preds = { }

        if "NormalizationStats" in self.json_info:
            batch = normalize_batch(batch, self.json_info)
        else:
            batch = batch/255.

        batch_output = self.model.learn.model(torch.tensor(batch).to(self.device).float())

        num_boxes = 0
        for chip_idx, output in enumerate(batch_output):
            pp_output = self.model._analyze_pred(pred=output, thresh=self.thres, nms_overlap=self.nms_overlap)
            image_bbox = _reconstruct(pp_output, dummy_x, pad_idx=0, classes=['background'] + class_names)
            if not image_bbox is None:
                for feature_idx in range(len(image_bbox.data[0])):
                    to_append = pred2dict(((image_bbox.data[0][feature_idx] + 1) / 2).detach().cpu().numpy(),
                                                image_bbox.scores[feature_idx],
                                                str(image_bbox.labels[feature_idx]),
                                                image_bbox.data[1][feature_idx],
                                                )

                    try:
                        preds[chip_idx].append(to_append)
                    except KeyError:
                        preds[chip_idx] = [to_append]

                    num_boxes += 1
            else:
                preds[chip_idx] = [{}]
                num_boxes += 1

        batch_size = self.batch_size
        side = math.sqrt(batch_size)

        bounding_boxes = np.zeros(shape=(num_boxes, 4), dtype=np.float)
        scores = np.zeros(shape=(num_boxes), dtype=np.float)
        classes = np.zeros(shape=(num_boxes), dtype=np.uint8)

        idx = 0
        tile_height = chip_size
        tile_width = chip_size

        for batch_idx in range(batch_size):
            i, j = batch_idx//side, batch_idx%side

            for pred in preds[batch_idx]:
                if pred == {}:
                    idx = idx+1
                    continue

                bounding_boxes[idx, 0] = (pred['y1'] + i)*tile_height
                bounding_boxes[idx, 1] = (pred['x1'] + j)*tile_width
                bounding_boxes[idx, 2] = (pred['y2'] + i)*tile_height
                bounding_boxes[idx, 3] = (pred['x2'] + j)*tile_width
                scores[idx] = pred['score']
                classes[idx] = pred['class']

                idx = idx+1

        return convert_bounding_boxes_to_coord_list(bounding_boxes), scores * 100, classes


import json

chemin = "/Users/vincentbataille/Desktop/formation/formation/python_fr/exo/"
fichier = "test.json"
my_file = chemin+fichier

# écrire dans le json
with open(my_file,"w") as f:
    json.dump(list(range(10)),f,indent=4)

with open(my_file,"w") as f:
    json.dump("pêche",f,ensure_ascii=False)

# lire le json
with open(my_file,"r") as f:
    liste=json.load(f)
    print(liste)


# sorting objects

class Employee:

    def __init__(self, name, age, pay):
        self.name = name
        self.age = age
        self.pay = pay

    def __repr__(self):  # always pass the self
        return 'Employee({}, {}, {})'.format(self.name, self.age, self.pay)

emp_1 = Employee('Corey', 50, 50000)
emp_2 = Employee('Ben', 33, 60000)
emp_3 = Employee('Jane', 23, 40000)

emloyees = [emp_1, emp_2, emp_3]

def e_sort(emp):
    return emp.age

# first way
s_employees = sorted(emloyees, key=e_sort, reverse=True)  # for comparing we have to define a function to pass to the key
print(s_employees)

# second way
s_employees = sorted(emloyees, key=lambda e: e.age)  # for such a short function we can use anonymous lambda function which is lambda arguments : expression
print(s_employees)

# third way
from operator import attrgetter
s_employees = sorted(emloyees, key=attrgetter('age'))  # for such a short function we can use anonymous lambda function which is lambda arguments : expression
print(s_employees)


import itertools
import operator
from collections import defaultdict
from functools import reduce

def count_by_attr(iterable, by):
    d = defaultdict(int)
    for obj in iterable:
        key = getattr(obj, by)
        d[key] += 1

    return d

def dict_by_attr(iterable, by, get=None, default_factory=None):
    if default_factory is not None:
        d = defaultdict(default_factory)
    else:
        d = {}

    for obj in iterable:
        key = getattr(obj, by)
        if get is not None:
            obj = getattr(obj, get)
        d[key] = obj

    return d

def group_by_attr(iterable, by, get=None):
    d = defaultdict(list)
    for obj in iterable:
        key = getattr(obj, by)
        if get is not None:
            obj = getattr(obj, get)
        d[key].append(obj)

    return d

def iter_compare_dicts(dict1, dict2, only_common_keys=False, comparison_op=operator.ne):
    """
    A generator for comparation of values in the given two dicts.

    Yields the tuples (key, pair of values positively compared).

    By default, the *difference* of values is evaluated using the usual != op, but can be changed
    by passing other comparison_op (a function of two arguments returning True/False).

    For example: operator.eq for equal values, operator.is_not for not identical objects.

    You can also require comparison only over keys existing in both dicts (only_common_keys=True).
    Otherwise, you will get the pair with the Python built-in Ellipsis placed for dict with
    that key missing. (Be sure to test for Ellipsis using the 'is' operator.)

    >>> d1 = dict(a=1, b=2, c=3)
    >>> d2 = dict(a=1, b=20, d=4)
    >>> dict(iter_compare_dicts(d1, d2, only_common_keys=True))
    {'b': (2, 20)}
    >>> dict(iter_compare_dicts(d1, d2, only_common_keys=True, comparison_op=operator.eq))
    {'a': (1, 1)}
    >>> dict(iter_compare_dicts(d1, d2))
    {'c': (3, Ellipsis), 'b': (2, 20), 'd': (Ellipsis, 4)}
    >>> dict(iter_compare_dicts(d1, d2, comparison_op=operator.eq))
    {'a': (1, 1), 'c': (3, Ellipsis), 'd': (Ellipsis, 4)}
    """
    keyset1, keyset2 = set(dict1), set(dict2)

    for key in (keyset1 & keyset2):
        pair = (dict1[key], dict2[key])
        if reduce(comparison_op, pair):
            yield key, pair

    if not only_common_keys:
        for key in (keyset1 - keyset2):
            yield key, (dict1[key], Ellipsis)
        for key in (keyset2 - keyset1):
            yield key, (Ellipsis, dict2[key])

def iter_ibatches(iterable, size):
    """
    https://code.activestate.com/recipes/303279-getting-items-in-batches/

    Generates iterators of elements of fixed size from the source iterable. Does not create batch sequences in memory.
    The source iterable can be of an unknown arbitrary length, does not need to support anything else than iteration.
    itertools.islice provides a size-bounded iterator over the given iterator.

    To know when we're done batching is the tricky part, as islice is happy to continue returning empty iterators
    on source exhaustion. We never want to yield an empty iterator. So we try to consume each batch a bit, which
    possibly raises StopIteration stopping the generator function itself.

    WARNING:
    Each batch must be entirely consumed before proceeding to the next one, otherwise you will get unexpected behaviour!

    >>> for b in iter_ibatches(range(55), 10):
    ...     print(tuple(b))
    (0, 1, 2, 3, 4, 5, 6, 7, 8, 9)
    (10, 11, 12, 13, 14, 15, 16, 17, 18, 19)
    (20, 21, 22, 23, 24, 25, 26, 27, 28, 29)
    (30, 31, 32, 33, 34, 35, 36, 37, 38, 39)
    (40, 41, 42, 43, 44, 45, 46, 47, 48, 49)
    (50, 51, 52, 53, 54)

    """
    it = iter(iterable)
    while True:
        batch_it = itertools.islice(it, size)
        try:
            yield itertools.chain([next(batch_it)], batch_it)
        except StopIteration:
            return


# -*- coding: utf-8 -*-
#import serial
import datetime
import json
import time
import os
message = [0, 0, 0, 0]
looper = 0

# Change event bitmask into list
# "01010" will change into [2,4]

def event_to_list(event):
    events = event[6:]
    event_list = []
    for idx, item in enumerate(events):
        if item == "1":
            event_list.append(idx+1)
    return event_list

def setup():
    global s
    global data, json_index
    global time_shift
    global alarm_dict
    global immediately_start_time

    # Handling timezone not in UTC+8
    # Define timezone here (UTC+8 for Taiwan)
    timezone = 8
    time_shift = (timezone*60*60)-(-time.timezone)

    # Try opening in serial port
    # Other wise output message to STDOUT

    #s = serial.Serial("/dev/ttyS0", 57600)
    #s.write("0000")

    # Read file from .json
    with open("/root/Schedule.json") as json_file:
        data = json.loads(json_file.read())

    # Turn events bitmask into dictionary structure
    # alarm_dict = { str event_start_time : [ int index, str event_end_time, list [ int event_number ], int position ] }
    # An event at 18:30 with event bitmask:10010210101 will turn into
    # {"17:30":[0,"18:30",[1,3,5],2]}
    # which index may change depend on real index
    json_index = list(data.keys())
    alarm_dict = {}
    for item in json_index:
        # Get event from json Ex : "10001210101"
        event = event_to_list(str(data[item]["events"]))
        # Calculate event start time
        # Ex: An event end at 18:30 will be break down into
        # hh="17"
        # mm="30"
        hh = str((datetime.datetime.strptime(
            str(data[item]["hour"]), "%H")-datetime.timedelta(hours=1)).hour)
        mm = str(data[item]["minute"])
        # Get position of which MXA7219 should light up
        # Ex: An event end at 18:30 will have position 2
        # Since minute 30 should be set at THIRD MXA7219(position 2)
        position = str(data[item]["events"])[1:5].index("1")
        alarm_dict[hh+":"+mm] = [item,
                                 str(data[item]["hour"])+":"+str(data[item]["minute"]), event, position]

    flag = ["0", "0", "0", "0"]
    # Determin if there are any event should wake up immediately
    # By setting flag with start time
    # For example, if now time is 18:30 and a 19:15 event should wake up immediately
    # Than flag will become ["0","19:15","0","0"]
    # If therer are multi event should wake up immediately
    # Than flag will be set up multi time.
    # Be aware of the item index been set up to event_start_time is the same as the position in alarm_dict
    # Position means which MXA7219 should be set up for.
    # In the above example, minutes 15 means second MXA7219 should be set up
    # which is the same as the position of event_start_time in flag.
    for times in alarm_dict:
        # Get Now time
        now = datetime.datetime.now()+datetime.timedelta(seconds=time_shift)
        ymd = str(now.year)+"-"+str(now.month)+"-"+str(now.day)
        # Get an event_start_time
        begin_time = datetime.datetime.strptime(
            ymd+" "+times+":"+'0', "%Y-%m-%d %H:%M:%S")
        # Get an event_end_time
        end_time = datetime.datetime.strptime(
            ymd+" "+alarm_dict[times][1]+":"+'0', "%Y-%m-%d %H:%M:%S")
        # Determint if there are any event_start_time between Now time and event_end_time
        # Than set flag to event_start_time
        if (begin_time-now).total_seconds() <= 0 and (end_time-now).total_seconds() >= 0:
            immediately_start_time = times
            flag[alarm_dict[times][3]] = immediately_start_time
    print(alarm_dict)
    print(flag)

    return flag

def loop(flag):
    global message
    global looper
    # Get Now time in hh:mm format
    now = datetime.datetime.now()+datetime.timedelta(seconds=time_shift)
    current_hour, current_minute = str(now.hour), str(now.minute)
    current_time = current_hour+":"+current_minute
    # Travel through all event
    # If current time match event in alarm_dict
    # Add it to flag
    for start_time in alarm_dict:
        if current_time == start_time:  # start of event
            flag[alarm_dict[start_time][3]] = current_time

    # Travel through flag and determint which event should be handle.
    for idx, start_time in enumerate(flag):
        # flag[idx]== "0" means there are current not event on that time.
        if start_time == "0":
            message[idx] = 0
        # Handle enent which is alive
        elif start_time != "0":
            # Handle NULL event
            if len(alarm_dict[start_time][2]) == 0:
                message[idx] = 0
            # Save an event to message
            else:
                message[idx] = alarm_dict[start_time][2][looper %
                                                         len(alarm_dict[start_time][2])]
            # If current_time is event_end_time, set flag[idx]="0" to clear event in flag
            if current_time == alarm_dict[start_time][1]:
                flag[idx] = "0"
    #Do a looper++, looper will help chosen an event in event list
    looper += 1
    looper %= 60

    # Bring up message and turn into a string
    s_message = str(message[0])+str(message[1])+str(message[2])+str(message[3])

    # Write message to serial port.
    #s.write((s_message))
    print(s_message)

    time.sleep(2)

if __name__ == '__main__':
    os.system("ntpd -q -p ptbtime1.ptb.de")
    flag = setup()
    cou=0
    while True:
        loop(flag)
    if cou==0:
        os.system("ntpd -q -p ptbtime1.ptb.de")
        cou+=1
        cou%=120


import numpy as np
import pandas as pd

def carrega_base_escore(base):
    base=("MIMICIII_TESTE_10000.csv")
    dataset2 = pd.read_csv("Data/"+base, sep=';')
    X_saida=[]
    Y_saida=[]
    X_tst=dataset2[[#'permanencia_uti',
        'acidbase_score', 'mingcs','hr_score', 'meanbp_score', 'temp_score',
        'resprate_score', 'hematocrit_score', 'wbc_score', 'creatinine_score', 'bun_score',
        'sodium_score', 'albumin_score', 'bilirubin_score', 'glucose_score', 'uo_score',
        'gcs_score', 'pao2_aado2_score', 'apsiii', 'morte_hosp']]
    X_tst[np.isnan(X_tst)] = 0
    features = list(X_tst.columns[:17])
    Target = X_tst.columns[18]
    y = X_tst[Target]
    X = X_tst[features]
    X_saida = [x for x in X.values]
    Y_saida= [x for x in y.values]

    return X_saida,Y_saida

#X,y=carrega_base_escore("mimic_test_des.csv")
#print(X)
#print(X)
 #   '''


# encoding=utf-8
from __future__ import unicode_literals
import unittest
from mwtextextractor import get_body_text

class TestMWTextExtractor(unittest.TestCase):

    def setUp(self):
        pass

    def test_simple_template(self):
        # Check that a simple template is stripped
        text = 'Alfa {{beta}} gamma'
        te = get_body_text(text).split()
        self.assertEqual(len(te), 2)  # only two words should remain

    def test_recursive_tables(self):
        # Check that recursive tables are stripped correctly
        text = 'Alfa {| \n|beta \n{| \n|gamma \n|} \n|} delta'
        te = get_body_text(text).split()
        self.assertEqual(len(te), 2)  # only two words should remain

    def test_math(self):
        # Check that a simple template is stripped
        text = 'on alkuperäinen suure ja <math>1/2 \le r<1 < 2<3</math>. Eudoksoksen oppilas'
        te = get_body_text(text).split()
        self.assertEqual(len(te), 6)

if __name__ == '__main__':
    unittest.main()


"""
Routes and views for the flask application.
"""

from __future__ import unicode_literals, print_function
from datetime import datetime
from flask import render_template
from SlideManager import app
from flask import request
import os, io

UPLOAD_FOLDER = "SlideManager/static/uploads/"

@app.route('/')
@app.route('/home')
def home():
    """Renders the home page."""
    return render_template(
        'index.html',
        title='Home Page',
        year=datetime.now().year,
    )

"""
@app.route('/contact')
def contact():
    return render_template(
        'contact.html',
        title='Contact',
        year=datetime.now().year,
        message='Your contact page.'
    )
"""
@app.route('/about')
def about():
    return render_template(
        'about.html',
        title='About',
        year=datetime.now().year,
        message='Your application description page.'
    )

@app.route('/new_presentation/',methods=["GET", "POST"])
def new_presentation():
    print("Add a presentation")
    foldername = request.data.decode("UTF-8")
    try:
        os.mkdir(UPLOAD_FOLDER + foldername)
        return "Success"
    except:
        return "Failed"
@app.route('/<string:presentation_name>/add_photo/', methods=["GET", "POST"])
def add_photo(presentation_name):
    print("Add a photo to", presentation_name)
    if presentation_name in os.listdir(UPLOAD_FOLDER):
        file = request.files['file']
        if file.filename.split("_")[0] == presentation_name:
            file.save(os.path.join(UPLOAD_FOLDER, presentation_name, file.filename))
            return "Success"
    return "Failed", 500

@app.route('/<string:presentation_name>/', methods=["GET", "POST"])
def show_slide(presentation_name):
    if presentation_name in os.listdir(UPLOAD_FOLDER):
        return io.open(os.path.join(UPLOAD_FOLDER, presentation_name, 'index.html'), encoding="utf-8").read()
    return "Failed", 500

@app.route('/<string:presentation_name>/main_html/', methods=["GET", "POST"])
def modify_html(presentation_name):
    print("Modifying HTML of", presentation_name)
    if presentation_name in os.listdir(UPLOAD_FOLDER):
        file = request.files['file']
        if ".".join(file.filename.split(".")[:-1]) == presentation_name:
            file.save(os.path.join(UPLOAD_FOLDER, presentation_name, 'index.html'))
            return presentation_name
    return "Failed", 500