# Copyright (C) 2019 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>

"""Test on Cycle Task status update over import."""

import datetime
from collections import OrderedDict

import mock
import ddt

from ggrc import db
from ggrc.models import all_models
from ggrc.converters import errors

from integration import ggrc as ggrc_test
from integration.ggrc_workflows.models import factories
from integration.ggrc.models import factories as ggrc_factories

DENY_FINISHED_DATES_STATUSES_STR = ("<'Assigned' / 'In Progress' / "
                                    "'Declined' / 'Deprecated'>")
DENY_VERIFIED_DATES_STATUSES_STR = ("<'Assigned' / 'In Progress' / "
                                    "'Declined' / 'Deprecated' / 'Finished'>")

@ddt.ddt
class TestCycleTaskStatusUpdate(ggrc_test.TestCase):
  """Test cases for update status on import cycle tasks"""

  @classmethod
  def setUpClass(cls):
    """Setup special hooks to that test class."""
    cls._current_user_wfo_or_assignee = (
        all_models.CycleTaskGroupObjectTask.current_user_wfa_or_assignee
    )
    all_models.CycleTaskGroupObjectTask.current_user_wfa_or_assignee = (
        mock.MagicMock(return_value=True))

  @classmethod
  def tearDownClass(cls):
    all_models.CycleTaskGroupObjectTask.current_user_wfa_or_assignee = (
        cls._current_user_wfo_or_assignee
    )

  DEFAULT_TASK_STATUSES = [all_models.CycleTaskGroupObjectTask.ASSIGNED] * 3

  ALIAS = "Cycle Task"

  def setUp(self):
    with ggrc_factories.single_commit():
      self.workflow = factories.WorkflowFactory(
          status=all_models.Workflow.ACTIVE
      )
      self.cycle = factories.CycleFactory(workflow=self.workflow)
      self.group = factories.CycleTaskGroupFactory(
          cycle=self.cycle,
          context=self.cycle.workflow.context
      )
      self.tasks = []
      for ind, task_status in enumerate(self.DEFAULT_TASK_STATUSES):
        self.tasks.append(factories.CycleTaskGroupObjectTaskFactory(
            title='task{}'.format(ind),
            cycle=self.cycle,
            cycle_task_group=self.group,
            context=self.cycle.workflow.context,
            status=task_status,
        ))
    # Emulate that current user is assignee for all test CycleTasks.
    self.task_ids = [t.id for t in self.tasks]

  VERIFIED_STRUCTURE = {
      "task_statuses": [all_models.CycleTaskGroupObjectTask.VERIFIED] * 3,
      "group_status": all_models.CycleTaskGroup.VERIFIED,
      "cycle_status": all_models.Cycle.VERIFIED,
      "workflow_status": all_models.Workflow.INACTIVE,
  }
  ASSIGNED_STRUCTURE = {
      "task_statuses": [all_models.CycleTaskGroupObjectTask.ASSIGNED] * 3,
      "group_status": all_models.CycleTaskGroup.IN_PROGRESS,
      "cycle_status": all_models.Cycle.IN_PROGRESS,
      "workflow_status": all_models.Workflow.ACTIVE
  }
  IN_PROGRESS_STRUCTURE = {
      "task_statuses": [all_models.CycleTaskGroupObjectTask.IN_PROGRESS] * 3,
      "group_status": all_models.CycleTaskGroup.IN_PROGRESS,
      "cycle_status": all_models.Cycle.IN_PROGRESS,
      "workflow_status": all_models.Workflow.ACTIVE
  }
  DECLINED_STRUCTURE = {
      "task_statuses": [all_models.CycleTaskGroupObjectTask.DECLINED] * 3,
      "group_status": all_models.CycleTaskGroup.IN_PROGRESS,
      "cycle_status": all_models.Cycle.IN_PROGRESS,
      "workflow_status": all_models.Workflow.ACTIVE,
  }
  FINISHED_STRUCTURE = {
      "task_statuses": [all_models.CycleTaskGroupObjectTask.FINISHED] * 3,
      "group_status": all_models.CycleTaskGroup.FINISHED,
      "cycle_status": all_models.Cycle.FINISHED,
      "workflow_status": all_models.Workflow.ACTIVE,
  }

  def _update_structure(self, structure=None):
    """Update structure of setuped objects."""
    if not structure:
      return
    structure = structure or {}
    start_statuses = structure.get("task_statuses") or []
    if start_statuses:
      for idx, status in enumerate(start_statuses):
        self.tasks[idx].status = status
    self.group.status = structure.get("group_status", self.group.status)
    self.cycle.status = structure.get("cycle_status", self.cycle.status)
    self.cycle.is_verification_needed = structure.get(
        "cycle_is_verification_needed",
        self.cycle.is_verification_needed)
    self.workflow.status = structure.get("workflow_status",
                                         self.workflow.status)
    db.session.commit()

  def _get_start_tasks_statuses(self, start_structure=None):
    start_statuses = (start_structure or {}).get("task_statuses", [])
    return (start_statuses + self.DEFAULT_TASK_STATUSES)[:3]

  def build_import_data(self, task_statuses):
    return [OrderedDict([("object_type", self.ALIAS),
                         ("Code*", self.tasks[idx].slug),
                         ("State", status)])
            for idx, status in enumerate(task_statuses)]

  @ddt.unpack
  @ddt.data(
      {"update_structure": VERIFIED_STRUCTURE},
      {"update_structure": IN_PROGRESS_STRUCTURE},
      {"update_structure": FINISHED_STRUCTURE},
      # verified few tasks
      {"update_structure": {
          "task_statuses": [all_models.CycleTaskGroupObjectTask.VERIFIED] * 2,
          "group_status": all_models.CycleTaskGroup.IN_PROGRESS,
          "cycle_status": all_models.Cycle.IN_PROGRESS,
          "workflow_status": all_models.Workflow.ACTIVE,
      }},
      {"update_structure": DECLINED_STRUCTURE},
      {"update_structure": VERIFIED_STRUCTURE,
       "start_structure": IN_PROGRESS_STRUCTURE},
      {"update_structure": IN_PROGRESS_STRUCTURE,
       "start_structure": VERIFIED_STRUCTURE},
      {"update_structure": DECLINED_STRUCTURE,
       "start_structure": IN_PROGRESS_STRUCTURE},
      {"update_structure": IN_PROGRESS_STRUCTURE,
       "start_structure": DECLINED_STRUCTURE},
  )
  def test_update_status(self, update_structure, start_structure=None):
    """Simple update task status to {update_structure[task_statuses]}"""
    self._update_structure(start_structure)
    start_statuses = self._get_start_tasks_statuses(start_structure)
    task_count = len(all_models.CycleTaskGroupObjectTask.query.all())
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    task_statuses = update_structure["task_statuses"]
    group_status = update_structure["group_status"]
    cycle_status = update_structure["cycle_status"]
    workflow_status = update_structure["workflow_status"]
    response = self.import_data(*self.build_import_data(task_statuses))
    self._check_csv_response(response, {})
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    self.assertEqual(task_statuses + start_statuses[len(task_statuses):],
                     [t.status for t in self.tasks])
    group = self.tasks[0].cycle_task_group
    self.assertEqual(group_status, group.status)
    self.assertEqual(len(all_models.CycleTaskGroupObjectTask.query.all()),
                     task_count)
    self.assertEqual(cycle_status, group.cycle.status)
    self.assertEqual(workflow_status, group.cycle.workflow.status)

  @ddt.data(ASSIGNED_STRUCTURE,
            IN_PROGRESS_STRUCTURE,
            DECLINED_STRUCTURE)
  def test_update_to_verified(self, start_structure):
    """Update task status to verified from {0[task_statuses]} and back"""
    self._update_structure(start_structure)
    start_statuses = self._get_start_tasks_statuses(start_structure)
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.verified_date for t in self.tasks])
    task_statuses = self.VERIFIED_STRUCTURE["task_statuses"]
    response = self.import_data(*self.build_import_data(task_statuses))
    self._check_csv_response(response, {})
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    self.assertEqual(task_statuses + start_statuses[len(task_statuses):],
                     [t.status for t in self.tasks])
    finished_dates = [t.finished_date for t in self.tasks]
    self.assertNotEqual([None] * len(self.tasks), finished_dates)
    verified_dates = [t.verified_date for t in self.tasks]
    self.assertNotEqual([None] * len(self.tasks), verified_dates)
    self.assertFalse(self.tasks[0].cycle.is_current)
    # wrong rollback  case (try to import from verified to finished)
    response = self.import_data(*self.build_import_data(start_statuses))
    line_error_tmpl_1 = errors.INVALID_STATUS_DATE_CORRELATION.format(
        date="Actual Verified Date",
        deny_states=DENY_VERIFIED_DATES_STATUSES_STR,
        line="{}"
    )
    line_error_tmpl_2 = errors.INVALID_STATUS_DATE_CORRELATION.format(
        date="Actual Finish Date",
        deny_states=DENY_FINISHED_DATES_STATUSES_STR,
        line="{}"
    )
    verfied_date_errors = {line_error_tmpl_1.format(i + 3)
                           for i in xrange(len(self.tasks))}
    finished_date_errors = {line_error_tmpl_2.format(i + 3)
                            for i in xrange(len(self.tasks))}
    self._check_csv_response(
        response,
        {
            "Cycle Task": {
                "row_errors": verfied_date_errors | finished_date_errors
            }
        }
    )
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    # nothing change
    self.assertEqual(task_statuses + start_statuses[len(task_statuses):],
                     [t.status for t in self.tasks])
    self.assertEqual(finished_dates, [t.finished_date for t in self.tasks])
    self.assertEqual(verified_dates, [t.verified_date for t in self.tasks])
    self.assertFalse(self.tasks[0].cycle.is_current)
    # correct rollback
    rollback_data = self.build_import_data(start_statuses)
    # setup verified dates as empty
    for line in rollback_data:
      line["Actual Verified Date"] = "--"
      line["Actual Finish Date"] = "--"
    response = self.import_data(*rollback_data)
    self._check_csv_response(response, {})
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    # assert correct rollback
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.verified_date for t in self.tasks])
    self.assertTrue(self.tasks[0].cycle.is_current)

  @ddt.data(1, 2, 3)
  def test_finished_to_verified(self, number_of_tasks):
    """Update {0} tasks status from finished from verified and back"""
    now = datetime.datetime.now().replace(microsecond=0)
    finished_date = now - datetime.timedelta(1)
    start_statuses = self._get_start_tasks_statuses(self.FINISHED_STRUCTURE)
    with ggrc_factories.single_commit():
      for task in self.tasks:
        task.finished_date = finished_date
    self._update_structure(self.FINISHED_STRUCTURE)
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    self.assertEqual([finished_date] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.verified_date for t in self.tasks])
    task_statuses = [
        all_models.CycleTaskGroupObjectTask.VERIFIED
    ] * number_of_tasks
    response = self.import_data(*self.build_import_data(task_statuses))
    self._check_csv_response(response, {})
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    self.assertEqual(task_statuses + start_statuses[len(task_statuses):],
                     [t.status for t in self.tasks])
    self.assertEqual([finished_date] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    verified_dates = [t.verified_date for t in self.tasks][:number_of_tasks]
    self.assertNotEqual([None] * number_of_tasks, verified_dates)
    verified_dates += [t.verified_date for t in self.tasks][number_of_tasks:]
    self.assertEqual(
        [None] * (len(self.tasks) - number_of_tasks),
        [t.verified_date for t in self.tasks][number_of_tasks:])
    self.assertEqual(len(self.tasks) != number_of_tasks,
                     self.tasks[0].cycle.is_current)
    # wrong rollback  case (try to import from verified to finished)
    response = self.import_data(*self.build_import_data(start_statuses))
    line_error_tmpl = errors.INVALID_STATUS_DATE_CORRELATION.format(
        date="Actual Verified Date",
        deny_states=DENY_VERIFIED_DATES_STATUSES_STR,
        line="{}"
    )
    self._check_csv_response(
        response,
        {
            "Cycle Task": {
                "row_errors": {line_error_tmpl.format(idx + 3)
                               for idx in xrange(number_of_tasks)}
            }
        }
    )
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    # nothing change
    self.assertEqual(task_statuses + start_statuses[len(task_statuses):],
                     [t.status for t in self.tasks])
    self.assertEqual([finished_date] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    self.assertEqual(len(self.tasks) != number_of_tasks,
                     self.tasks[0].cycle.is_current)
    self.assertEqual(
        verified_dates,
        [t.verified_date for t in self.tasks])
    # correct rollback
    rollback_data = self.build_import_data(start_statuses)
    # setup verified dates as empty
    for line in rollback_data:
      line["Actual Verified Date"] = "--"
    response = self.import_data(*rollback_data)
    self._check_csv_response(response, {})
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    # assert correct rollback
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    self.assertEqual([finished_date] * len(self.tasks),
                     [t.finished_date for t in self.tasks])
    self.assertEqual([None] * len(self.tasks),
                     [t.verified_date for t in self.tasks])
    self.assertTrue(self.tasks[0].cycle.is_current)

  @staticmethod
  def __build_error_tmpl(error_tmpl, **context):
    """Create simple temaplte from base error tmpl."""
    if "line" not in context:
      context["line"] = "{}"
    return error_tmpl.format(**context)

  def __build_status_error_resp(self, key, error_tmpl, exception_statuses):
    """Return expected response dict based on sent arguments."""
    error_tmpl = self.__build_error_tmpl(error_tmpl,
                                         column_name="State",
                                         message="Invalid state '{}'")
    return {
        self.ALIAS: {
            key: {error_tmpl.format(3 + idx, status)
                  for idx, status in enumerate(exception_statuses)},
        }
    }

  def __assert_error_message(self,
                             sending_data,
                             expected_messages,
                             start_structure=None):
    """Assert validation error message on import cycle task statuses."""
    start_statuses = self._get_start_tasks_statuses(start_structure)

    group_status = self.group.status
    cycle_status = self.cycle.status
    workflow_status = self.workflow.status
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    response = self.import_data(*sending_data)
    self._check_csv_response(response, expected_messages)
    self.tasks = all_models.CycleTaskGroupObjectTask.query.filter(
        all_models.CycleTaskGroupObjectTask.id.in_(self.task_ids)
    ).all()
    group = self.tasks[0].cycle_task_group
    self.assertEqual(start_statuses, [t.status for t in self.tasks])
    self.assertEqual(group_status, group.status)
    self.assertEqual(cycle_status, group.cycle.status)
    self.assertEqual(workflow_status, group.cycle.workflow.status)

  @ddt.data(
      VERIFIED_STRUCTURE["task_statuses"],
      DECLINED_STRUCTURE["task_statuses"],
  )
  def test_validation_error(self, exception_statuses):
    """Validation cycle task status update to {0} if no verification needed."""
    self._update_structure({"cycle_is_verification_needed": False})
    error_msgs = self.__build_status_error_resp("row_errors",
                                                errors.VALIDATION_ERROR,
                                                exception_statuses)
    send_data = self.build_import_data(exception_statuses)
    self.__assert_error_message(send_data, error_msgs)

  @ddt.unpack
  @ddt.data(
      {"start_structure": IN_PROGRESS_STRUCTURE,
       "exception_statuses": ["Absolutely Wrong Status"] * 3},
      {"start_structure": IN_PROGRESS_STRUCTURE,
       "exception_statuses": [""] * 3},
      {"exception_statuses": [""] * 3},
      {"exception_statuses": ["Absolutely Wrong Status"] * 3},
      {"start_structure": FINISHED_STRUCTURE,
       "exception_statuses": ["Absolutely Wrong Status"] * 3},
      {"start_structure": FINISHED_STRUCTURE,
       "exception_statuses": [""] * 3},
  )
  def test_simple_send_invalid_status(self,
                                      exception_statuses,
                                      start_structure=None):
    """Validation cycle task status update to {exception_statuses}."""
    self._update_structure(start_structure)
    warn_msgs = self.__build_status_error_resp("row_warnings",
                                               errors.WRONG_VALUE_CURRENT,
                                               exception_statuses)
    send_data = self.build_import_data(exception_statuses)
    self.__assert_error_message(send_data,
                                warn_msgs,
                                start_structure=start_structure)

  @ddt.data(
      ("Actual Finish Date", "Actual Verified Date"),
      ("Start Date", "Due Date"),
  )
  def test_for_date_compare_error(self, columns):
    """Validate task import data where {0[0]} bigger than {0[1]}."""
    self._update_structure(self.VERIFIED_STRUCTURE)
    today = datetime.date.today()
    dates = (today, today - datetime.timedelta(7))
    error_tmpl = self.__build_error_tmpl(errors.INVALID_START_END_DATES,
                                         start_date=columns[0],
                                         end_date=columns[1])
    # line format
    error_resp = {self.ALIAS: {"row_errors": {error_tmpl.format(3)}}}
    data_to_import = OrderedDict([("object_type", self.ALIAS),
                                  ("Code*", self.tasks[0].slug)])
    for column, date in zip(columns, dates):
      data_to_import[column] = date
    response = self.import_data(data_to_import)
    self._check_csv_response(response, error_resp)

  @ddt.data(
      ("Actual Finish Date", DENY_FINISHED_DATES_STATUSES_STR),
      ("Actual Verified Date", DENY_VERIFIED_DATES_STATUSES_STR),
  )
  @ddt.unpack
  def test_for_date_state_error(self, column, deny_states):
    """Validate task {0} not allowed for in {0} tasks."""
    self._update_structure(self.IN_PROGRESS_STRUCTURE)
    today = datetime.date.today()
    error_tmpl = self.__build_error_tmpl(
        errors.INVALID_STATUS_DATE_CORRELATION,
        date=column,
        deny_states=deny_states,
    )
    # line format
    error_resp = {self.ALIAS: {"row_errors": {error_tmpl.format(3)}}}
    data_to_import = OrderedDict([("object_type", self.ALIAS),
                                  ("Code*", self.tasks[0].slug),
                                  (column, today)])
    response = self.import_data(data_to_import)
    self._check_csv_response(response, error_resp)


# Copyright (C) 2017 Beijing Didi Infinity Technology and Development Co.,Ltd.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Transformer layers."""
import math

from absl import logging
import numpy as np
import tensorflow as tf
from tensorflow.python.util import nest

from delta.layers.base_layer import Layer

# pylint: disable=invalid-name, too-many-instance-attributes, too-many-arguments, too-many-locals

SOS_ID = 4
EOS_ID = 5
INF = 1. * 1e7

def log_prob_from_logits(logits, reduce_axis=-1):
  """return log prob use log sum func"""
  return logits - tf.reduce_logsumexp(logits, axis=reduce_axis, keepdims=True)

def shape_list(tensor):
  """Return list of dims, statically where possible."""
  tensor = tf.convert_to_tensor(tensor)

  if tensor.get_shape().dims is None:
    return tf.shape(tensor)

  static = tensor.get_shape().as_list()
  shape = tf.shape(tensor)

  ret = []
  for i, _ in enumerate(static):
    dim = static[i]
    if dim is None:
      dim = shape[i]
    ret.append(dim)
  return ret

def _merge_beam_dim(tensor):
  """Reshapes first two dimensions in to single dimension."""
  shape = shape_list(tensor)
  shape[0] *= shape[1]  # batch -> batch * beam_size
  shape.pop(1)  # Remove beam dim
  return tf.reshape(tensor, shape)

def _unmerge_beam_dim(tensor, batch_size, beam_size):
  """Reshapes first dimension back to [batch_size, beam_size]."""
  shape = shape_list(tensor)
  new_shape = [batch_size] + [beam_size] + shape[1:]
  return tf.reshape(tensor, new_shape)

def _expand_to_beam_size(tensor, beam_size):
  """Tiles a given tensor by beam_size."""
  tensor = tf.expand_dims(tensor, axis=1)
  tile_dims = [1] * tensor.shape.ndims
  tile_dims[1] = beam_size

  return tf.tile(tensor, tile_dims)

def get_state_shape_invariants(tensor):
  """Returns the shape of the tensor but sets middle dims to None."""
  shape = tensor.shape.as_list()
  for i in range(1, len(shape) - 1):
    shape[i] = None
  return tf.TensorShape(shape)

def compute_batch_indices(batch_size, beam_size):
  """Computes the i'th coordinate that contains the batch index for gathers."""
  batch_pos = tf.range(batch_size * beam_size) // beam_size
  batch_pos = tf.reshape(batch_pos, [batch_size, beam_size])
  return batch_pos

def _create_make_unique(inputs):
  """Replaces the lower bits of each element with iota."""
  if inputs.shape.ndims != 2:
    raise ValueError("Input of top_k_with_unique must be rank-2 "
                     "but got: %s" % inputs.shape)

  height = inputs.shape[0]
  width = inputs.shape[1]
  zeros = tf.zeros([height, width], dtype=tf.int32)

  log2_ceiling = int(math.ceil(math.log(int(width), 2)))
  next_power_of_two = 1 << log2_ceiling
  count_mask = ~(next_power_of_two - 1)
  count_mask_r0 = tf.constant(count_mask)
  count_mask_r2 = tf.fill([height, width], count_mask_r0)

  smallest_normal = 1 << 23
  smallest_normal_r0 = tf.constant(smallest_normal, dtype=tf.int32)
  smallest_normal_r2 = tf.fill([height, width], smallest_normal_r0)

  low_bit_mask = ~(1 << 31)
  low_bit_mask_r0 = tf.constant(low_bit_mask, dtype=tf.int32)
  low_bit_mask_r2 = tf.fill([height, width], low_bit_mask_r0)

  iota = tf.tile(
      tf.expand_dims(tf.range(width, dtype=tf.int32), 0), [height, 1])

  input_r2 = tf.bitcast(inputs, tf.int32)
  abs_r2 = tf.bitwise.bitwise_and(input_r2, low_bit_mask_r2)
  if_zero_r2 = tf.equal(abs_r2, zeros)
  smallest_normal_preserving_sign_r2 = tf.bitwise.bitwise_or(
      input_r2, smallest_normal_r2)
  input_no_zeros_r2 = tf.where(if_zero_r2, smallest_normal_preserving_sign_r2,
                               input_r2)

  and_r2 = tf.bitwise.bitwise_and(input_no_zeros_r2, count_mask_r2)
  or_r2 = tf.bitwise.bitwise_or(and_r2, iota)
  return tf.bitcast(or_r2, tf.float32)

def _create_topk_unique(inputs, k):
  """Creates the top k values in sorted order with indices."""
  height = inputs.shape[0]
  width = inputs.shape[1]
  neg_inf_r0 = tf.constant(-np.inf, dtype=tf.float32)
  ones = tf.ones([height, width], dtype=tf.float32)
  neg_inf_r2 = ones * neg_inf_r0
  inputs = tf.where(tf.is_nan(inputs), neg_inf_r2, inputs)

  tmp = inputs
  topk_r2 = tf.zeros([height, k], dtype=tf.float32)
  for i in range(k):
    kth_order_statistic = tf.reduce_max(tmp, axis=1, keepdims=True)
    k_mask = tf.tile(
        tf.expand_dims(tf.equal(tf.range(k), tf.fill([k], i)), 0), [height, 1])
    topk_r2 = tf.where(k_mask, tf.tile(kth_order_statistic, [1, k]), topk_r2)
    ge_r2 = tf.greater_equal(inputs, tf.tile(kth_order_statistic, [1, width]))
    tmp = tf.where(ge_r2, neg_inf_r2, inputs)

  log2_ceiling = int(math.ceil(math.log(float(int(width)), 2)))
  next_power_of_two = 1 << log2_ceiling
  count_mask = next_power_of_two - 1
  mask_r0 = tf.constant(count_mask)
  mask_r2 = tf.fill([height, k], mask_r0)
  topk_r2_s32 = tf.bitcast(topk_r2, tf.int32)
  topk_indices_r2 = tf.bitwise.bitwise_and(topk_r2_s32, mask_r2)
  return topk_r2, topk_indices_r2

def top_k_with_unique(inputs, k):
  """Finds the values and indices of the k largests entries."""
  unique_inputs = _create_make_unique(tf.cast(inputs, tf.float32))
  top_values, indices = _create_topk_unique(unique_inputs, k)
  top_values = tf.cast(top_values, inputs.dtype)
  return top_values, indices

def compute_topk_scores_and_seq(sequences,
                                scores,
                                scores_to_gather,
                                flags,
                                beam_size,
                                batch_size,
                                prefix="default",
                                states_to_gather=None):
  """Given sequences and scores, will gather the top k=beam size sequences."""
  _, topk_indexes = tf.nn.top_k(scores, k=beam_size)
  batch_pos = compute_batch_indices(batch_size, beam_size)
  top_coordinates = tf.stack([batch_pos, topk_indexes], axis=2)

  def gather(tensor, name):
    return tf.gather_nd(tensor, top_coordinates, name=(prefix + name))

  topk_seq = gather(sequences, "_topk_seq")
  topk_flags = gather(flags, "_topk_flags")
  topk_gathered_scores = gather(scores_to_gather, "_topk_scores")
  if states_to_gather:
    topk_gathered_states = nest.map_structure(
        lambda state: gather(state, "_topk_states"), states_to_gather)
  else:
    topk_gathered_states = states_to_gather

  return topk_seq, topk_gathered_scores, topk_flags, topk_gathered_states

class MultiHeadAttention(Layer):
  """
  Refer to [Attention Is All You Need]
    (https://arxiv.org/abs/1706.03762)
  Input shape: (Batch size, steps, features)
  Output shape: (Batch size, steps, features)
  """

  def __init__(self,
               head_num,
               activation='relu',
               use_bias=False,
               kernel_initializer='glorot_normal',
               bias_initializer='zeros',
               kernel_regularizer=None,
               bias_regularizer=None,
               kernel_constraint=None,
               bias_constraint=None,
               sequence_mask=False,
               **kwargs):

    super().__init__(**kwargs)
    self.supports_masking = True
    self.head_num = head_num
    self.activation = tf.keras.activations.get(activation)
    self.use_bias = use_bias
    self.sequence_mask = sequence_mask
    self.kernel_initializer = tf.keras.initializers.get(kernel_initializer)
    self.bias_initializer = tf.keras.initializers.get(bias_initializer)
    self.kernel_regularizer = tf.keras.regularizers.get(kernel_regularizer)
    self.bias_regularizer = tf.keras.regularizers.get(bias_regularizer)
    self.kernel_constraint = tf.keras.constraints.get(kernel_constraint)
    self.bias_constraint = tf.keras.constraints.get(bias_constraint)

  @staticmethod
  def _unpack(inputs):
    """unpack attention inputs, shape and mask"""
    query, key, value = inputs
    return query, key, value

  def build(self, input_shape):
    # pylint: disable=attribute-defined-outside-init
    query_shape, key_shape, value_shape = self._unpack(input_shape)

    feature_dim = int(value_shape[-1])
    if feature_dim % self.head_num != 0:
      error_info = 'Invalid head number {} with the given input dim {}'.format(
          self.head_num, feature_dim)
      logging.error(error_info)
      raise ValueError(error_info)

    self.kernel_query = self.add_weight(
        name='Wq',
        shape=(int(query_shape[-1]), feature_dim),
        initializer=self.kernel_initializer,
        regularizer=self.kernel_regularizer,
        constraint=self.kernel_constraint)
    self.kernel_key = self.add_weight(
        name='Wk',
        shape=(int(key_shape[-1]), feature_dim),
        initializer=self.kernel_initializer,
        regularizer=self.kernel_regularizer,
        constraint=self.kernel_constraint)
    self.kernel_value = self.add_weight(
        name='Wv',
        shape=(int(value_shape[-1]), feature_dim),
        initializer=self.kernel_initializer,
        regularizer=self.kernel_regularizer,
        constraint=self.kernel_constraint)

    self.kernel_project = self.add_weight(
        name='Wo',
        shape=(feature_dim, feature_dim),
        initializer=self.kernel_initializer,
        regularizer=self.kernel_regularizer,
        constraint=self.kernel_constraint)

    if self.use_bias:
      self.b_query = self.add_weight(
          name='Bq',
          shape=(feature_dim,),
          initializer=self.kernel_initializer,
          regularizer=self.kernel_regularizer,
          constraint=self.kernel_constraint)

      self.b_key = self.add_weight(
          name='Bk',
          shape=(feature_dim,),
          initializer=self.kernel_initializer,
          regularizer=self.kernel_regularizer,
          constraint=self.kernel_constraint)

      self.b_value = self.add_weight(
          name='Bb',
          shape=(feature_dim,),
          initializer=self.kernel_initializer,
          regularizer=self.kernel_regularizer,
          constraint=self.kernel_constraint)

      self.b_project = self.add_weight(
          name='Bo',
          shape=(feature_dim,),
          initializer=self.kernel_initializer,
          regularizer=self.kernel_regularizer,
          constraint=self.kernel_constraint)

  def compute_mask(self, inputs, mask=None):
    query_mask, _, _ = self._unpack(mask)
    return query_mask

  def compute_output_shape(self, input_shape):
    query_shape, _, value_shape = self._unpack(input_shape)
    return query_shape[:-1] + (value_shape[-1],)

  def call(self, inputs, training=None, mask=None):

    query, key, value = self._unpack(inputs)

    query_mask, key_mask, _ = self._unpack(mask)

    batch_size = tf.shape(query)[0]
    dimension_query = query.get_shape().as_list()[-1]
    seq_len = tf.shape(query)[-2]
    key_len = tf.shape(key)[-2]
    feature_dim = tf.shape(value)[-1]

    query = tf.matmul(
        query,
        tf.tile(tf.expand_dims(self.kernel_query, 0), [batch_size, 1, 1]))
    key = tf.matmul(
        key, tf.tile(tf.expand_dims(self.kernel_key, 0), [batch_size, 1, 1]))
    value = tf.matmul(
        value,
        tf.tile(tf.expand_dims(self.kernel_value, 0), [batch_size, 1, 1]))
    if self.use_bias:
      query += self.b_query
      key += self.b_key
      value += self.b_value

    def _reshape_multihead(origin_input):
      """
      reshape for multi head
        Input shape: (Batch size, steps, features)
        Output shape: (Batch size * head num, steps, features // head num)
      """
      return tf.concat(tf.split(origin_input, self.head_num, axis=2), axis=0)

    def _reshape_mask(mask):
      """
      repeat mask for multi head
        Input shape: (Batch size, steps)
        Output shape: (Batch size * head num, steps)
      """
      if mask is None:
        return None
      seq_len = tf.shape(mask)[1]
      mask = tf.expand_dims(mask, axis=1)
      mask = tf.tile(mask, [1, self.head_num, 1])
      return tf.reshape(mask, shape=(-1, seq_len))

    query_ = _reshape_multihead(query)
    key_ = _reshape_multihead(key)
    value_ = _reshape_multihead(value)

    key_mask = _reshape_mask(key_mask)

    # (Batch size * head num, query steps, key steps)
    similaritys = tf.matmul(query_, tf.transpose(key_, [0, 2, 1]))
    # scale
    similaritys /= tf.sqrt(tf.cast(dimension_query, tf.float32))
    if self.sequence_mask:
      ones = tf.ones((seq_len, key_len))
      similaritys -= (ones - tf.matrix_band_part(ones, -1, 0)) * 1e9
    if key_mask is not None:
      similaritys -= (
          1.0 - tf.cast(tf.expand_dims(key_mask, axis=-2), tf.float32)) * 1e9

    attention_weights = tf.keras.activations.softmax(similaritys)
    attention_outputs = tf.matmul(attention_weights, value_)
    attention_outputs = tf.reshape(
        attention_outputs,
        (-1, self.head_num, seq_len, feature_dim // self.head_num))
    attention_outputs = tf.transpose(attention_outputs, [0, 2, 1, 3])
    attention_outputs = tf.reshape(attention_outputs,
                                   (-1, seq_len, feature_dim))

    attention_outputs = tf.matmul(
        attention_outputs,
        tf.tile(tf.expand_dims(self.kernel_project, 0), [batch_size, 1, 1]))
    if self.use_bias:
      attention_outputs += self.b_project
    if self.activation is not None:
      attention_outputs = self.activation(attention_outputs)

    if query_mask is not None:
      attention_outputs *= tf.cast(
          tf.expand_dims(query_mask, axis=-1), tf.float32)

    return attention_outputs

class MultiHeadSelfAttention(MultiHeadAttention):
  """
  Refer to [Attention Is All You Need]
    (https://arxiv.org/abs/1706.03762)
  Input shape: (Batch size, steps, features)
  Output shape: (Batch size, steps, features)
  """

  @staticmethod
  def _unpack(inputs):
    """unpack attention inputs, shape and mask"""
    query = key = value = inputs
    return query, key, value

class LayerNormalization(Layer):
  """
  Refer to [Layer Normalization]
    (https://arxiv.org/pdf/1607.06450.pdf)
    Input shape: (Batch size, steps, features)
    Output shape: (Batch size, steps, features)
  """

  def __init__(self, eps=1e-8, **kwargs):
    self.eps = eps
    self.gamma, self.beta = None, None
    super().__init__(**kwargs)

  def build(self, input_shape):
    self.gamma = self.add_weight(
        name='gamma',
        shape=input_shape[-1:],
        initializer=tf.keras.initializers.ones,
        trainable=True)
    self.beta = self.add_weight(
        name='beta',
        shape=input_shape[-1:],
        initializer=tf.keras.initializers.zeros,
        trainable=True)
    super().build(input_shape)

  def call(self, inputs, training=None, mask=None):
    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)
    normalized = (inputs - mean) / ((variance + self.eps)**.5)
    return self.gamma * normalized + self.beta

  def compute_output_shape(self, input_shape):
    return input_shape

class PositionEmbedding(Layer):
  """
  Position embedding use sine and cosine functions
  Refer to [Attention Is All You Need]
    (https://arxiv.org/abs/1706.03762)
    Input shape: (Batch size, steps, features)
    Output shape: (Batch size, steps, features)
  """

  def __init__(self, max_len, embedding_dim, **kwargs):
    self.max_len = max_len
    self.embedding_dim = embedding_dim
    self.pos_embedding_matrix = self.get_pos_embedding_matrix(
        self.max_len, self.embedding_dim)
    embed_initializer = tf.constant_initializer(self.pos_embedding_matrix)
    self.pos_embedding_layer = tf.keras.layers.Embedding(
        *self.pos_embedding_matrix.shape,
        trainable=False,
        embeddings_initializer=embed_initializer)
    self.get_pos_layer = tf.keras.layers.Lambda(self.get_pos)
    self.mask_layer = tf.keras.layers.Lambda(self.mask_outputs)
    super().__init__(**kwargs)

  @staticmethod
  def get_pos_embedding_matrix(max_len, embedding_dim):
    """get position embedding by sine and cosine functions"""
    # First part of the PE function: sin and cos argument
    position_enc = np.array([[
        pos / np.power(10000, (i - i % 2) / embedding_dim)
        for i in range(embedding_dim)
    ]
                             for pos in range(max_len)])

    # Second part, apply the cosine to even columns and sin to odds.
    position_enc[:, 0::2] = np.sin(position_enc[:, 0::2])  # dim 2i
    position_enc[:, 1::2] = np.cos(position_enc[:, 1::2])  # dim 2i+1
    return position_enc

  @staticmethod
  def get_pos(inputs):
    """get position id"""
    batch_size, seq_len = tf.shape(inputs)[0], tf.shape(inputs)[1]
    position_ind = tf.tile(
        tf.expand_dims(tf.range(seq_len), 0), [batch_size, 1])
    return position_ind

  @staticmethod
  def mask_outputs(origin_outputs):
    """mask position embedding"""
    inputs, outputs = origin_outputs
    outputs = tf.where(tf.equal(inputs, 0), inputs, outputs)
    return outputs

  def call(self, inputs, training=None, mask=None):
    pos_ind = self.get_pos_layer(inputs)
    pos_embedding = self.pos_embedding_layer(pos_ind)
    pos_embedding = self.mask_layer([inputs, pos_embedding])
    return pos_embedding

class TransformerEncoderLayer(Layer):
  """
  Transformer Encoder Block Layer
  Refer to [Attention Is All You Need]
    (https://arxiv.org/abs/1706.03762)
  Input shape: (Batch size, steps, features)
  Output shape: (Batch size, steps, features)
  """

  def __init__(self, config, **kwargs):
    model_config = config['model']['net']['structure']
    self.head_num = model_config.get('head_num')
    self.hidden_dim = model_config.get('hidden_dim')
    self.feature_dim = model_config.get('embedding_size')
    self.attention_activation = config.get('attention_activation', None)
    self.feed_forward_activation = config.get('feed_forward_activation', 'relu')
    self.dropout_rate = config.get('transformer_dropout', 0.)
    self.residual_conn = config.get('residual_conn', False)

    self.attention_layer = MultiHeadSelfAttention(self.head_num,
                                                  self.attention_activation)
    self.attention_dropout_layer = tf.keras.layers.Dropout(self.dropout_rate)
    self.hidden_layer = tf.keras.layers.Conv1D(
        self.hidden_dim, 1, activation=self.feed_forward_activation)
    self.feed_forward_layer = tf.keras.layers.Conv1D(self.feature_dim, 1)
    self.feed_forward_dropout = tf.keras.layers.Dropout(self.dropout_rate)
    if self.residual_conn:
      self.attention_layernorm = LayerNormalization()
      self.feed_forward_layernorm = LayerNormalization()
    super().__init__(**kwargs)

  def call(self, inputs, training=None, mask=None):
    '''
    Input shape: (batch_size, enc_len, feature_dim)
    Mask shape: (batch_size, enc_len)
    '''
    # Multi Head Attention
    attention_layer = self.attention_layer(inputs, training=training, mask=mask)
    attention_dropout_layer = self.attention_dropout_layer(
        attention_layer, training=training)
    if self.residual_conn:
      attention_out = tf.keras.layers.add([inputs, attention_dropout_layer])
      attention_out = self.attention_layernorm(attention_out)
    else:
      attention_out = attention_dropout_layer

    # Position Wise Feed Forward
    hidden_layer = self.hidden_layer(attention_out)
    feed_forward_layer = self.feed_forward_layer(hidden_layer)
    feed_forward_dropout = self.feed_forward_dropout(
        feed_forward_layer, training=training)
    if self.residual_conn:
      feed_forward_out = tf.keras.layers.add(
          [attention_out, feed_forward_dropout])
      feed_forward_out = self.feed_forward_layernorm(feed_forward_out)
    else:
      feed_forward_out = feed_forward_layer

    return feed_forward_out

class TransformerDecoderLayer(Layer):
  """
  Transformer Decoder Block Layer
  Refer to [Attention Is All You Need]
    (https://arxiv.org/abs/1706.03762)
  Input shape: (Batch size, steps, features)
  Output shape: (Batch size, steps, features)
  """

  def __init__(self, config, **kwargs):
    model_config = config['model']['net']['structure']
    self.head_num = model_config.get('head_num')
    self.hidden_dim = model_config.get('hidden_dim')
    self.feature_dim = model_config.get('embedding_size')
    self.attention_activation = config.get('attention_activation', None)
    self.feed_forward_activation = config.get('feed_forward_activation', 'relu')
    self.dropout_rate = config.get('transformer_dropout', 0.)

    # Self Attention
    self.self_attention_layer = MultiHeadSelfAttention(
        self.head_num, self.attention_activation, sequence_mask=True)
    self.self_attention_dropout_layer = tf.keras.layers.Dropout(
        self.dropout_rate)
    self.self_attention_ln = LayerNormalization()

    # Context Attention
    self.context_attention_layer = MultiHeadAttention(self.head_num,
                                                      self.attention_activation)
    self.context_attention_dropout_layer = tf.keras.layers.Dropout(
        self.dropout_rate)
    self.context_attention_ln = LayerNormalization()

    # Position Wise Feed Forward
    self.hidden_layer = tf.keras.layers.Conv1D(
        self.hidden_dim, 1, activation=self.feed_forward_activation)
    self.feed_forward_layer = tf.keras.layers.Conv1D(self.feature_dim, 1)
    self.feed_forward_dropout = tf.keras.layers.Dropout(self.dropout_rate)
    self.feed_forward_ln = LayerNormalization()

    super().__init__(**kwargs)

  def call(self, inputs, training=None, mask=None):
    '''
    Input: [decoder_inputs, encoder_outputs]
    Mask: [decoder_mask, encoder_mask]

    Input shape: [(batch_size, dec_len, feature_dim), (batch_size, enc_len, feature_dim)]
    Mask shape: [(batch_size, dec_len), (batch_size, enc_len)]
    '''
    decoder_inputs, encoder_outputs = inputs
    decoder_mask, encoder_mask = mask
    # Self
    self_attention = self.self_attention_layer(
        decoder_inputs, training=training, mask=decoder_mask)
    self_attention_dropout = self.self_attention_dropout_layer(
        self_attention, training=training)
    self_attention_out = tf.keras.layers.add(
        [decoder_inputs, self_attention_dropout])
    self_attention_out = self.self_attention_ln(self_attention_out)

    # Context Attention
    context_attention = self.context_attention_layer(
        [self_attention_out, encoder_outputs, encoder_outputs],
        mask=[decoder_mask, encoder_mask, encoder_mask],
        training=training)
    context_attention_dropout = self.context_attention_dropout_layer(
        context_attention, training=training)
    context_attention_out = tf.keras.layers.add(
        [self_attention_out, context_attention_dropout])
    context_attention_out = self.self_attention_ln(context_attention_out)

    # Position Wise Feed Forward
    hidden_layer = self.hidden_layer(context_attention_out)
    feed_forward_layer = self.feed_forward_layer(hidden_layer)
    feed_forward_dropout = self.feed_forward_dropout(
        feed_forward_layer, training=training)
    feed_forward_out = tf.keras.layers.add(
        [context_attention_out, feed_forward_dropout])
    feed_forward_out = self.feed_forward_ln(feed_forward_out)

    return feed_forward_out

class TransformerEncoder(Layer):
  """
  Transformer Encoder Layer
  Refer to [Attention Is All You Need]
    (https://arxiv.org/abs/1706.03762)
  Input shape: (Batch size, steps, features)
  Output shape: (Batch size, steps, features)
  """

  def __init__(self, config, **kwargs):
    model_config = config['model']['net']['structure']
    self.is_infer = config['model']['is_infer']
    if self.is_infer:
      self.length_penalty = model_config['length_penalty']
    self.dropout_rate = model_config['dropout_rate']
    self.embedding_size = model_config['embedding_size']
    self.num_layers = model_config['num_layers']
    self.l2_reg_lambda = model_config['l2_reg_lambda']
    self.max_enc_len = model_config['max_enc_len']
    self.max_dec_len = model_config['max_dec_len']
    self.share_embedding = model_config['share_embedding']
    self.padding_token = 0
    self.beam_size = model_config['beam_size']

    self.transformer_encoders = [
        TransformerEncoderLayer(config) for _ in range(self.num_layers)
    ]

    super().__init__(**kwargs)

  def call(self, inputs, training=None, mask=None):
    enc_inp = inputs
    for encoder_layer in self.transformer_encoders:
      enc_inp = encoder_layer(enc_inp, training=training, mask=mask)
    enc_out = enc_inp
    return enc_out

class TransformerDecoder(Layer):
  """
  Transformer Decoder Layer
  Refer to [Attention Is All You Need]
    (https://arxiv.org/abs/1706.03762)
  Input shape: (Batch size, steps, features)
  Output shape: (Batch size, vocab_size)
  """

  def __init__(self, config, emb_layer, vocab_size, **kwargs):
    model_config = config['model']['net']['structure']
    self.is_infer = config['model']['is_infer']
    if self.is_infer:
      self.length_penalty = model_config['length_penalty']
    self.dropout_rate = model_config['dropout_rate']
    self.num_layers = model_config['num_layers']
    self.l2_reg_lambda = model_config['l2_reg_lambda']
    self.embedding_size = model_config['embedding_size']
    self.max_enc_len = model_config['max_enc_len']
    self.max_dec_len = model_config['max_dec_len']
    self.share_embedding = model_config['share_embedding']
    self.padding_token = 0
    self.beam_size = model_config['beam_size']

    self.mask_layer = tf.keras.layers.Lambda(lambda inputs: tf.cast(
        tf.not_equal(inputs, self.padding_token), tf.int32))

    self.embed = emb_layer
    self.vocab_size = vocab_size
    self.embed_d = tf.keras.layers.Dropout(self.dropout_rate)

    self.pos_embed = PositionEmbedding(self.max_enc_len, self.embedding_size)

    self.transformer_decoders = [
        TransformerDecoderLayer(config) for _ in range(self.num_layers)
    ]

    self.final_dense = tf.keras.layers.TimeDistributed(
        tf.keras.layers.Dense(self.vocab_size, name="final_dense"))

    super().__init__(**kwargs)

  def decode(self, input_dec_x, enc_out, enc_mask, training=None):
    """
    Decoder func
    """
    dec_mask = self.mask_layer(input_dec_x)
    dec_emb = self.embed(input_dec_x)
    dec_pos_emb = self.pos_embed(dec_emb)
    dec_emb = tf.keras.layers.add([dec_emb, dec_pos_emb])

    dec_inp = dec_emb
    for decoder_layer in self.transformer_decoders:
      dec_inp = decoder_layer([dec_inp, enc_out],
                              training=training,
                              mask=[dec_mask, enc_mask])
    dec_out = dec_inp
    return dec_out

  def call(self, inputs, training=None, mask=None):
    if not self.is_infer:
      dec_inp, enc_out = inputs
      with tf.name_scope('while'):
        dec_out = self.decode(dec_inp, enc_out, mask, training)
        scores = self.final_dense(dec_out)
        return scores
    else:
      enc_out = inputs
      init_ids = tf.cast(tf.ones([tf.shape(enc_out)[0]]) * SOS_ID, tf.int32)
      # Beam Search
      enc_shape = shape_list(enc_out)
      enc_out = tf.tile(
          tf.expand_dims(enc_out, axis=1), [1, self.beam_size, 1, 1])
      enc_mask = tf.tile(tf.expand_dims(mask, axis=1), [1, self.beam_size, 1])
      enc_out = tf.reshape(
          enc_out, [enc_shape[0] * self.beam_size, enc_shape[1], enc_shape[2]])
      enc_mask = tf.reshape(enc_mask,
                            [enc_shape[0] * self.beam_size, enc_shape[1]])

      def symbols_to_logits_fn(dec_inputs):
        dec_out = self.decode(dec_inputs, enc_out, enc_mask, training)
        scores = self.final_dense(dec_out)
        return scores[:, -1, :]

      decoded_ids, scores, _ = self.beam_search(symbols_to_logits_fn, init_ids,
                                                self.beam_size,
                                                self.max_dec_len,
                                                self.vocab_size,
                                                self.length_penalty)
      decoded_ids = decoded_ids[:, 0, 1:]

      return decoded_ids

  @staticmethod
  def beam_search(symbols_to_logits_fn,
                  initial_ids,
                  beam_size,
                  decode_length,
                  vocab_size,
                  alpha,
                  states=None,
                  eos_id=EOS_ID,
                  stop_early=True):
    """Beam search with length penalties."""
    batch_size = shape_list(initial_ids)[0]

    initial_log_probs = tf.constant([[0.] + [-INF] * (beam_size - 1)])
    # Expand to beam_size (batch_size, beam_size)
    # (batch_size, beam_size)
    alive_log_probs = tf.tile(initial_log_probs, [batch_size, 1])

    # Expand each batch and state to beam_size
    alive_seq = _expand_to_beam_size(initial_ids, beam_size)
    # (batch_size, beam_size, 1)
    alive_seq = tf.expand_dims(alive_seq, axis=2)
    if states:
      states = nest.map_structure(
          lambda state: _expand_to_beam_size(state, beam_size), states)
    else:
      states = {}

    # (batch_size, beam_size, 1)
    finished_seq = tf.zeros(shape_list(alive_seq), tf.int32)
    # Setting the scores of the initial to negative infinity.
    # (batch_size, beam_size)
    finished_scores = tf.ones([batch_size, beam_size]) * -INF
    # (batch_size, beam_size)
    finished_flags = tf.zeros([batch_size, beam_size], tf.bool)

    def grow_finished(finished_seq, finished_scores, finished_flags, curr_seq,
                      curr_scores, curr_finished):
      """
        Given sequences and scores from finished sequence and current finished sequence
        , will gather the top k=beam size sequences to update finished seq.
      """
      # padding zero for finished seq
      finished_seq = tf.concat(
          [finished_seq,
           tf.zeros([batch_size, beam_size, 1], tf.int32)],
          axis=2)

      # mask unfinished curr seq
      curr_scores += (1. - tf.to_float(curr_finished)) * -INF

      # concatenating the sequences and scores along beam axis
      # (batch_size, 2xbeam_size, seq_len)
      curr_finished_seq = tf.concat([finished_seq, curr_seq], axis=1)
      curr_finished_scores = tf.concat([finished_scores, curr_scores], axis=1)
      curr_finished_flags = tf.concat([finished_flags, curr_finished], axis=1)
      return compute_topk_scores_and_seq(curr_finished_seq,
                                         curr_finished_scores,
                                         curr_finished_scores,
                                         curr_finished_flags, beam_size,
                                         batch_size, "grow_finished")

    def grow_alive(curr_seq, curr_scores, curr_log_probs, curr_finished,
                   states):
      """Given sequences and scores, will gather the top k=beam size sequences."""
      curr_scores += tf.to_float(curr_finished) * -INF
      return compute_topk_scores_and_seq(curr_seq, curr_scores, curr_log_probs,
                                         curr_finished, beam_size, batch_size,
                                         "grow_alive", states)

    def grow_topk(i, alive_seq, alive_log_probs, states):
      """Inner beam search loop."""

      flat_ids = tf.reshape(alive_seq, [batch_size * beam_size, -1])

      # (batch_size * beam_size, decoded_length)
      if states:
        flat_states = nest.map_structure(_merge_beam_dim, states)
        flat_logits, flat_states = symbols_to_logits_fn(flat_ids, i,
                                                        flat_states)
        states = nest.map_structure(
            lambda t: _unmerge_beam_dim(t, batch_size, beam_size), flat_states)
      else:
        flat_logits = symbols_to_logits_fn(flat_ids)

      logits = tf.reshape(flat_logits, [batch_size, beam_size, -1])

      candidate_log_probs = log_prob_from_logits(logits)

      log_probs = candidate_log_probs + tf.expand_dims(alive_log_probs, axis=2)

      length_penalty = tf.pow(((5. + tf.to_float(i + 1)) / 6.), alpha)

      curr_scores = log_probs / length_penalty
      flat_curr_scores = tf.reshape(curr_scores, [-1, beam_size * vocab_size])

      topk_scores, topk_ids = tf.nn.top_k(flat_curr_scores, k=beam_size * 2)

      topk_log_probs = topk_scores * length_penalty

      topk_beam_index = topk_ids // vocab_size
      topk_ids %= vocab_size  # Unflatten the ids
      batch_pos = compute_batch_indices(batch_size, beam_size * 2)
      topk_coordinates = tf.stack([batch_pos, topk_beam_index], axis=2)

      topk_seq = tf.gather_nd(alive_seq, topk_coordinates)
      if states:
        states = nest.map_structure(
            lambda state: tf.gather_nd(state, topk_coordinates), states)
      topk_seq = tf.concat([topk_seq, tf.expand_dims(topk_ids, axis=2)], axis=2)

      topk_finished = tf.equal(topk_ids, eos_id)

      return topk_seq, topk_log_probs, topk_scores, topk_finished, states

    def inner_loop(i, alive_seq, alive_log_probs, finished_seq, finished_scores,
                   finished_flags, states):
      """Inner beam search loop."""
      topk_seq, topk_log_probs, topk_scores, topk_finished, states = grow_topk(
          i, alive_seq, alive_log_probs, states)
      alive_seq, alive_log_probs, _, states = grow_alive(
          topk_seq, topk_scores, topk_log_probs, topk_finished, states)
      finished_seq, finished_scores, finished_flags, _ = grow_finished(
          finished_seq, finished_scores, finished_flags, topk_seq, topk_scores,
          topk_finished)

      return (i + 1, alive_seq, alive_log_probs, finished_seq, finished_scores,
              finished_flags, states)

    def _is_finished(i, unused_alive_seq, alive_log_probs, unused_finished_seq,
                     finished_scores, unused_finished_in_finished,
                     unused_states):
      """Checking termination condition.
      """
      max_length_penalty = tf.pow(((5. + tf.to_float(decode_length)) / 6.),
                                  alpha)
      lower_bound_alive_scores = alive_log_probs[:, 0] / max_length_penalty

      if not stop_early:
        lowest_score_of_finished_in_finished = tf.reduce_min(finished_scores)
      else:
        lowest_score_of_finished_in_finished = tf.reduce_max(
            finished_scores, axis=1)

      bound_is_met = tf.reduce_all(
          tf.greater(lowest_score_of_finished_in_finished,
                     lower_bound_alive_scores))

      return tf.logical_and(
          tf.less(i, decode_length), tf.logical_not(bound_is_met))

    inner_shape = tf.TensorShape([None, None, None])

    state_struc = nest.map_structure(get_state_shape_invariants, states)
    (_, alive_seq, alive_log_probs, finished_seq, finished_scores,
     finished_flags, states) = tf.while_loop(
         _is_finished,
         inner_loop, [
             tf.constant(0), alive_seq, alive_log_probs, finished_seq,
             finished_scores, finished_flags, states
         ],
         shape_invariants=[
             tf.TensorShape([]), inner_shape,
             alive_log_probs.get_shape(), inner_shape,
             finished_scores.get_shape(),
             finished_flags.get_shape(), state_struc
         ],
         parallel_iterations=1,
         back_prop=False)

    alive_seq.set_shape((None, beam_size, None))
    finished_seq.set_shape((None, beam_size, None))
    finished_seq = tf.where(
        tf.reduce_any(finished_flags, 1), finished_seq, alive_seq)
    finished_scores = tf.where(
        tf.reduce_any(finished_flags, 1), finished_scores, alive_log_probs)
    return finished_seq, finished_scores, states


#
# BSD 3-Clause License
#
# Copyright (c) 2017 xxxx
# All rights reserved.
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ============================================================================
#
import numpy as np
import logging
import pathlib
import xml.etree.ElementTree as ET
import cv2
import os

class VOCDataset:

    def __init__(self, root, transform=None, target_transform=None, is_test=False, keep_difficult=False, label_file=None):
        """Dataset for VOC data.
        Args:
            root: the root of the VOC2007 or VOC2012 dataset, the directory contains the following sub-directories:
                Annotations, ImageSets, JPEGImages, SegmentationClass, SegmentationObject.
        """
        self.root = pathlib.Path(root)
        self.transform = transform
        self.target_transform = target_transform
        if is_test:
            image_sets_file = self.root / "ImageSets/Main/test.txt"
        else:
            image_sets_file = self.root / "ImageSets/Main/trainval.txt"
        self.ids = VOCDataset._read_image_ids(image_sets_file)
        self.keep_difficult = keep_difficult

        # if the labels file exists, read in the class names
        label_file_name = self.root / "labels.txt"

        if os.path.isfile(label_file_name):
            class_string = ""
            with open(label_file_name, 'r') as infile:
                for line in infile:
                    class_string += line.rstrip()

            # classes should be a comma separated list

            classes = class_string.split(',')
            # prepend BACKGROUND as first class
            classes.insert(0, 'BACKGROUND')
            classes  = [ elem.replace(" ", "") for elem in classes]
            self.class_names = tuple(classes)
            logging.info("VOC Labels read from file: " + str(self.class_names))

        else:
            logging.info("No labels file, using default VOC classes.")
            self.class_names = ('BACKGROUND',
            'aeroplane', 'bicycle', 'bird', 'boat',
            'bottle', 'bus', 'car', 'cat', 'chair',
            'cow', 'diningtable', 'dog', 'horse',
            'motorbike', 'person', 'pottedplant',
            'sheep', 'sofa', 'train', 'tvmonitor')

        self.class_dict = {class_name: i for i, class_name in enumerate(self.class_names)}

    def __getitem__(self, index):
        image_id = self.ids[index]
        boxes, labels, is_difficult = self._get_annotation(image_id)
        if not self.keep_difficult:
            boxes = boxes[is_difficult == 0]
            labels = labels[is_difficult == 0]
        image = self._read_image(image_id)
        if self.transform:
            image, boxes, labels = self.transform(image, boxes, labels)
        if self.target_transform:
            boxes, labels = self.target_transform(boxes, labels)
        return image, boxes, labels

    def get_image(self, index):
        image_id = self.ids[index]
        image = self._read_image(image_id)
        if self.transform:
            image, _ = self.transform(image)
        return image

    def get_annotation(self, index):
        image_id = self.ids[index]
        return image_id, self._get_annotation(image_id)

    def __len__(self):
        return len(self.ids)

    @staticmethod
    def _read_image_ids(image_sets_file):
        ids = []
        with open(image_sets_file) as f:
            for line in f:
                ids.append(line.rstrip())
        return ids

    def _get_annotation(self, image_id):
        annotation_file = self.root / f"Annotations/{image_id}.xml"
        objects = ET.parse(annotation_file).findall("object")
        boxes = []
        labels = []
        is_difficult = []
        for object in objects:
            class_name = object.find('name').text.lower().strip()
            # we're only concerned with clases in our list
            if class_name in self.class_dict:
                bbox = object.find('bndbox')

                # VOC dataset format follows Matlab, in which indexes start from 0
                x1 = float(bbox.find('xmin').text) - 1
                y1 = float(bbox.find('ymin').text) - 1
                x2 = float(bbox.find('xmax').text) - 1
                y2 = float(bbox.find('ymax').text) - 1
                boxes.append([x1, y1, x2, y2])

                labels.append(self.class_dict[class_name])
                is_difficult_str = object.find('difficult').text
                is_difficult.append(int(is_difficult_str) if is_difficult_str else 0)

        return (np.array(boxes, dtype=np.float32),
                np.array(labels, dtype=np.int64),
                np.array(is_difficult, dtype=np.uint8))

    def _read_image(self, image_id):
        image_file = self.root / f"JPEGImages/{image_id}.jpg"
        image = cv2.imread(str(image_file))
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        return image


"""With the output of script_consistency.R, get figure 5."""

import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams.update({'font.size': 12})

if not os.path.exists("figures"): os.mkdir("figures")

###############################################################################
# GET CORRECT VECTOR FOR PLOTTING

def method_name_func(model, strategy, withpattern):
    method_name = ""
    if strategy == "mia": method_name = "MIA"
    elif strategy == "mean": method_name = "Mean imputation"
    elif strategy == "gaussian": method_name = "Gaussian imputation"
    elif strategy == "none":
        if model == "rpart": method_name = "Surrogates (rpart)"
        elif model == "ctree": method_name = "ctree (surrogates)"
        elif model == "xgboost": method_name = "Block (XGBoost)"
    if withpattern == "TRUE": method_name += " + mask"
    if model == "ranger": method_name += " - forest"
    if model == "xgboost": method_name += " - xgboost"
    return method_name

# remove the three first values: too small sample sizes
sizes = np.logspace(2, 5, 20).astype('int64')[3:]

scores_raw = []
# trees
for i in range(3):
    scorerawi = {}
    dataset="make_data{}".format(i+1)
    for method in [
        {'model': 'rpart', 'strategy': 'none', 'withpattern': "FALSE"},
        {'model': 'rpart', 'strategy': 'mean', 'withpattern': "FALSE"},
        {'model': 'rpart', 'strategy': 'gaussian', 'withpattern': "FALSE"},
        {'model': 'rpart', 'strategy': 'mia', 'withpattern': "FALSE"},
        ]:
        model, strategy, withpattern = tuple(method.values())

        csvfile = "consistency/results/{}_{}_{}_{}.csv".format(dataset, model, strategy, withpattern)
        if os.path.exists(csvfile):
            scores_method = pd.read_csv(csvfile, sep=" ")
            method_name = method_name_func(model, strategy, withpattern)
            scorerawi[method_name] = np.array(scores_method)[:,3:]
    scores_raw.append(scorerawi)

# forests
for i in range(3):
    scorerawi = {}
    dataset="make_data{}".format(i+1)
    for method in [
        {'model': 'ranger', 'strategy': 'mean', 'withpattern': "FALSE"},
        {'model': 'ranger', 'strategy': 'gaussian', 'withpattern': "FALSE"},
        {'model': 'ranger', 'strategy': 'mia', 'withpattern': "FALSE"},
        ]:
        model, strategy, withpattern = tuple(method.values())

        csvfile = "consistency/results/{}_{}_{}_{}.csv".format(dataset, model, strategy, withpattern)
        if os.path.exists(csvfile):
            scores_method = pd.read_csv(csvfile, sep=" ")
            method_name = method_name_func(model, strategy, withpattern)
            scorerawi[method_name] = np.array(scores_method)[:,3:]
    scores_raw.append(scorerawi)

# xgboost
for i in range(3):
    scorerawi = {}
    dataset="make_data{}".format(i+1)
    for method in [
        {'model': 'xgboost', 'strategy': 'mean', 'withpattern': "FALSE"},
        {'model': 'xgboost', 'strategy': 'gaussian', 'withpattern': "FALSE"},
        {'model': 'xgboost', 'strategy': 'mia', 'withpattern': "FALSE"},
        {'model': 'xgboost', 'strategy': 'none', 'withpattern': "FALSE"},
        ]:
        model, strategy, withpattern = tuple(method.values())

        csvfile = "consistency/results/{}_{}_{}_{}.csv".format(dataset, model, strategy, withpattern)
        if os.path.exists(csvfile):
            scores_method = pd.read_csv(csvfile, sep=" ")
            method_name = method_name_func(model, strategy, withpattern)
            scorerawi[method_name] = np.array(scores_method)[:,3:]
    scores_raw.append(scorerawi)

# Transformation of scores, mse to explained variance
VARY = [25, 1702, 10823.94, 25, 1702, 10823.94, 25, 1702, 10823.94]
scores_expvar = [{key: 1 - scr/VARY[datanum]
    for key, scr in score.items()} for datanum, score in enumerate(scores_raw)]

# sufficient statistics
scores = [{key: (np.percentile(val, 50, 0), np.percentile(val, 25, 0), np.percentile(val, 75, 0))
    for key, val in score.items()} for score in scores_expvar]

# these values come from bayesrate.R
L = len(sizes)
scores[0]['Bayes rate'] = (np.repeat(0.8087995, L), np.repeat(0, L), np.repeat(0, L))
scores[1]['Bayes rate'] = (np.repeat(0.7484916, L), np.repeat(0, L), np.repeat(0, L))
scores[3]['Bayes rate'] = (np.repeat(0.8087995, L), np.repeat(0, L), np.repeat(0, L))
scores[4]['Bayes rate'] = (np.repeat(0.7484916, L), np.repeat(0, L), np.repeat(0, L))
scores[6]['Bayes rate'] = (np.repeat(0.8087995, L), np.repeat(0, L), np.repeat(0, L))
scores[7]['Bayes rate'] = (np.repeat(0.7484916, L), np.repeat(0, L), np.repeat(0, L))

###############################################################################
###############################################################################
###############################################################################
# PLOT

plt.clf()
fig, ax = plt.subplots(3, 3, figsize=(10, 10))
colors = plt.rcParams['axes.prop_cycle'].by_key()['color']
colors_dict = {
    'Surrogates (rpart)': colors[0],
    'Mean imputation': colors[1],
    'Block (XGBoost) - xgboost': colors[5],
    'MIA': colors[2],
    'MIA - forest': colors[2],
    'MIA - xgboost': colors[2],
    'Gaussian imputation': colors[3],
    'Mean imputation - forest': colors[1],
    'Mean imputation - xgboost': colors[1],
    'Gaussian imputation - forest': colors[3],
    'Gaussian imputation - xgboost': colors[3],
    'Bayes rate': colors[4],
}
###############################################################################
# There are 3 x 3 plots: trees, forests, boosting for models 1, 2, 3
###############################################################################
for name, scr in scores[0].items():
    means = np.array(scr[0])
    q1 = np.array(scr[1])
    q2 = np.array(scr[2])
    if '+ mask' in name:
        linestyle = ':'
    else:
        linestyle = '-'
    ax[0,0].semilogx(sizes, means, label=name, linestyle=linestyle, c=colors_dict[name])
    ax[0,0].fill_between(sizes, q1, q2, alpha=0.2)
ax[0,0].set_xlabel("Sample size")
ax[0,0].set_ylabel("Explained variance")
ax[0,0].set_ylim(
    np.array([s[0] for _, s in scores[0].items()]).min()-0.05,
    np.array([s[0] for _, s in scores[0].items()]).max()+0.05
)
ax[0,0].set_xlim(297.63514416, 10**5)
ax[0,0].text(10**3, 0.95, "Linear problem\n(high noise)")
ax[0,0].grid()
###############################################################################
for name, scr in scores[1].items():
    means = np.array(scr[0])
    q1 = np.array(scr[1])
    q2 = np.array(scr[2])
    if '+ mask' in name:
        linestyle = ':'
    else:
        linestyle = '-'
    ax[0,1].semilogx(sizes, means, label=None, linestyle=linestyle, c=colors_dict[name])
    ax[0,1].fill_between(sizes, q1, q2, alpha=0.2)
ax[0,1].set_xlabel("Sample size")
ax[0,1].set_ylabel("Explained variance")
ax[0,1].set_ylim(
    np.array([s[0] for _, s in scores[1].items()]).min()-0.05,
    np.array([s[0] for _, s in scores[1].items()]).max()+0.05
)
ax[0,1].set_xlim(297.63514416, 10**5)
ax[0,1].text(10**3, 0.88, "Friedman problem\n(high noise)")
ax[0,1].grid()
###############################################################################
for name, scr in scores[2].items():
    means = np.array(scr[0])
    q1 = np.array(scr[1])
    q2 = np.array(scr[2])
    if '+ mask' in name:
        linestyle = ':'
    else:
        linestyle = '-'
    ax[0,2].semilogx(sizes, means, label=None, linestyle=linestyle, c=colors_dict[name])
    ax[0,2].fill_between(sizes, q1, q2, alpha=0.2)
ax[0,2].set_xlabel("Sample size")
ax[0,2].set_ylabel("Explained variance")
ax[0,2].set_ylim(
    np.array([s[0] for _, s in scores[2].items()]).min()-0.01,
    np.array([s[0] for _, s in scores[2].items()]).max()+0.05
)
ax[0,2].set_xlim(297.63514416, 10**5)
ax[0,2].text(10**3, 1.12, "Non-linear problem\n(low noise)")
ax[0,2].text(2*10**5, 0.95, "DECISION TREE", rotation=-90, fontweight='bold')
ax[0,2].grid()
###############################################################################
for name, scr in scores[3].items():
    means = np.array(scr[0])
    q1 = np.array(scr[1])
    q2 = np.array(scr[2])
    if '+ mask' in name:
        linestyle = ':'
    else:
        linestyle = '-'
    ax[1,0].semilogx(sizes, means, label=None, linestyle=linestyle, c=colors_dict[name])
    ax[1,0].fill_between(sizes, q1, q2, alpha=0.2)
ax[1,0].set_xlabel("Sample size")
ax[1,0].set_ylabel("Explained variance")
ax[1,0].set_ylim(
    np.array([s[0] for _, s in scores[3].items()]).min()-0.01,
    np.array([s[0] for _, s in scores[3].items()]).max()+0.01
)
ax[1,0].set_xlim(297.63514416, 10**5)
ax[1,0].grid()
###############################################################################
for name, scr in scores[4].items():
    means = np.array(scr[0])
    q1 = np.array(scr[1])
    q2 = np.array(scr[2])
    if '+ mask' in name:
        linestyle = ':'
    else:
        linestyle = '-'
    ax[1,1].semilogx(sizes, means, label=None, linestyle=linestyle, c=colors_dict[name])
    ax[1,1].fill_between(sizes, q1, q2, alpha=0.2)
ax[1,1].set_xlabel("Sample size")
ax[1,1].set_ylabel("Explained variance")
ax[1,1].set_ylim(
    np.array([s[0] for _, s in scores[4].items()]).min()-0.015,
    np.array([s[0] for _, s in scores[4].items()]).max()+0.015
)
ax[1,1].set_xlim(297.63514416, 10**5)
ax[1,1].grid()
###############################################################################
for name, scr in scores[5].items():
    means = np.array(scr[0])
    q1 = np.array(scr[1])
    q2 = np.array(scr[2])
    if '+ mask' in name:
        linestyle = ':'
    else:
        linestyle = '-'
    ax[1,2].semilogx(sizes, means, label=None, linestyle=linestyle, c=colors_dict[name])
    ax[1,2].fill_between(sizes, q1, q2, alpha=0.2)
ax[1,2].set_xlabel("Sample size")
ax[1,2].set_ylabel("Explained variance")
ax[1,2].set_ylim(
    np.array([s[0] for _, s in scores[5].items()]).min()-0.005,
    np.array([s[0] for _, s in scores[5].items()]).max()+0.005
)
ax[1,2].set_xlim(297.63514416, 10**5)
ax[1,2].text(2*10**5, 0.993, "RANDOM FOREST", rotation=-90, fontweight='bold')
ax[1,2].grid()
###############################################################################
for name, scr in scores[6].items():
    means = np.array(scr[0])
    q1 = np.array(scr[1])
    q2 = np.array(scr[2])
    if '+ mask' in name:
        linestyle = ':'
    else:
        linestyle = '-'
    if name == 'Block (XGBoost) - xgboost':
        ax[2,0].semilogx(sizes, means, label='Block (XGBoost)', linestyle=linestyle, c=colors_dict[name])
    else:
        ax[2,0].semilogx(sizes, means, label=None, linestyle=linestyle, c=colors_dict[name])

    ax[2,0].fill_between(sizes, q1, q2, alpha=0.2)
ax[2,0].set_xlabel("Sample size")
ax[2,0].set_ylabel("Explained variance")
ax[2,0].set_ylim(
    np.array([s[0] for _, s in scores[6].items()]).min()-0.01,
    np.array([s[0] for _, s in scores[6].items()]).max()+0.01
)
ax[2,0].set_xlim(297.63514416, 10**5)
ax[2,0].grid()
###############################################################################
for name, scr in scores[7].items():
    means = np.array(scr[0])
    q1 = np.array(scr[1])
    q2 = np.array(scr[2])
    if '+ mask' in name:
        linestyle = ':'
    else:
        linestyle = '-'
    ax[2,1].semilogx(sizes, means, label=None, linestyle=linestyle, c=colors_dict[name])
    ax[2,1].fill_between(sizes, q1, q2, alpha=0.2)
ax[2,1].set_xlabel("Sample size")
ax[2,1].set_ylabel("Explained variance")
ax[2,1].set_ylim(
    np.array([s[0] for _, s in scores[7].items()]).min()-0.015,
    np.array([s[0] for _, s in scores[7].items()]).max()+0.015
)
ax[2,1].set_xlim(297.63514416, 10**5)
ax[2,1].grid()
###############################################################################
for name, scr in scores[8].items():
    means = np.array(scr[0])
    q1 = np.array(scr[1])
    q2 = np.array(scr[2])
    if '+ mask' in name:
        linestyle = ':'
    else:
        linestyle = '-'
    ax[2,2].semilogx(sizes, means, label=None, linestyle=linestyle, c=colors_dict[name])
    ax[2,2].fill_between(sizes, q1, q2, alpha=0.2)
ax[2,2].set_xlabel("Sample size")
ax[2,2].set_ylabel("Explained variance")
ax[2,2].set_ylim(
    np.array([s[0] for _, s in scores[8].items()]).min()-0.005,
    np.array([s[0] for _, s in scores[8].items()]).max()+0.005
)
ax[2,2].set_xlim(297.63514416, 10**5)
ax[2,2].text(2*10**5, 0.993, "XGBOOST", rotation=-90, fontweight='bold')
ax[2,2].grid()
###############################################################################
fig.legend(loc=(0.17, 0.01), ncol=3, frameon=False)
plt.tight_layout()
fig.subplots_adjust(bottom=0.13, top=0.9, right=0.95)
fig.savefig('figures/consistency_log_merge.pdf')
plt.close(fig)


#!/usr/bin/env python

from setuptools import setup, find_packages

readme = open('README.rst').read()

long_description = "%s" % readme

setup(
    name='pysparklines',
    version=0.9,
    description="pysparklines is a unicode sparkline generation library.",
    long_description=long_description,
    author="Brandon Whaley",
    author_email="redkrieg@gmail.com",
    url="https://github.com/RedKrieg/pysparklines",
    packages=find_packages(),
    entry_points={
        'console_scripts': [
            'sparkline = sparkline:main',
        ]
    },
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Console',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'License :: OSI Approved :: BSD License',
        'Operating System :: Unix',
        'Operating System :: POSIX',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2.7',
        'Topic :: Software Development',
        'Topic :: Software Development :: Libraries',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
)


# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import shlex
from ssh import SSH
from subprocess import Popen, PIPE, call
from tempfile import NamedTemporaryFile
from os.path import basename
from constants import Constants as con
from robot.api import logger

__all__ = ["SetupFramework"]

class SetupFramework(object):
    """Setup suite run on topology nodes.

    Many VAT/CLI based tests need the scripts at remote hosts before executing
    them. This class packs the whole testing directory and copies it over
    to all nodes in topology under /tmp/

    """

    def __init__(self):
        pass

    def __pack_framework_dir(self):
        """Pack the testing WS into temp file, return its name."""

        tmpfile = NamedTemporaryFile(suffix=".tgz", prefix="openvpp-testing-")
        file_name = tmpfile.name
        tmpfile.close()

        proc = Popen(shlex.split("tar -zcf {0} .".format(file_name)),
                stdout=PIPE, stderr=PIPE)
        (stdout, stderr) = proc.communicate()

        logger.debug(stdout)
        logger.debug(stderr)

        return_code = proc.wait()
        if 0 != return_code:
            raise Exception("Could not pack testing framework.")

        return file_name

    def __copy_tarball_to_node(self, tarball, node):
        logger.console('Copying tarball to {0}'.format(node['host']))
        ssh = SSH()
        ssh.connect(node)

        ssh.scp(tarball, "/tmp/")

    def __extract_tarball_at_node(self, tarball, node):
        logger.console('Extracting tarball to {0} on {1}'.format(
            con.REMOTE_FW_DIR, node['host']))
        ssh = SSH()
        ssh.connect(node)

        cmd = 'rm -rf {1}; mkdir {1} ; sudo -Sn tar -zxf {0} -C {1};'.format(
                tarball, con.REMOTE_FW_DIR)
        (ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout=30)
        if 0 != ret_code:
            logger.error('Unpack error: {0}'.format(stderr))
            raise Exception('Failed to unpack {0} at node {1}'.format(
                tarball, node['host']))

    def __delete_local_tarball(self, tarball):
        call(shlex.split('sh -c "rm {0} > /dev/null 2>&1"'.format(tarball)))

    def setup_framework(self, nodes):
        """Pack the whole directory and extract in temp on each node."""

        tarball = self.__pack_framework_dir()
        logger.console('Framework packed to {0}'.format(tarball))
        remote_tarball = "/tmp/{0}".format(basename(tarball))

        for node in nodes.values():
            self.__copy_tarball_to_node(tarball, node)
            self.__extract_tarball_at_node(remote_tarball, node)

        logger.trace('Test framework copied to all topology nodes')
        self.__delete_local_tarball(tarball)


f = open("/tmp/foo.txt", "w")

f.write( "Python 是一个非常好的语言。\n是的，的确非常好!!\n" )

# 关闭打开的文件
f.close()

# 打开一个文件
f = open("/tmp/foo.txt", "r")
# 调用 f.read(size), 这将读取一定数目的数据, 然后作为字符串或字节对象返回。
# 当 size 被忽略了或者为负, 那么该文件的所有内容都将被读取并且返回。
str1 = f.read()
print(str1)

# 关闭打开的文件
f.close()

# f.readline()从文件中读取单独的一行。换行符为 '\n'。f.readline()
# 如果返回一个空字符串, 说明已经已经读取到最后一行。
f=open("/tmp/foo.txt","r")
str1=f.readline()
print(str1)
f.close()

# f.readlines()返回该文件中包含的所有行。
# 如果设置可选参数 sizehint, 则读取指定长度的字节, 并且将这些字节按行分割。

f = open("/tmp/foo.txt", "r")

str1 = f.readlines()
print(str1)

# 关闭打开的文件
f.close()

# 另一种方式是迭代一个文件对象然后读取每行:
f=open("/tmp/foo.txt","r")
for line in f :
    print(line,end=' ')
f.close()

# f.write(string) 将 string 写入到文件中, 然后返回写入的字符数。
f=open("/tmp/foo.txt","w")
num=f.write("Python 是一个非常好的语言. \n是的,的确非常好!!\n")
print(num)
f.close()

# 如果要写入一些不是字符串的东西, 那么将需要先进行转换:
f=open("/tmp/foo1.txt","w")
value =('www.runoob.com',14)
s=str(value)
f.write(s)
f.close()

f=open('/tmp/foo.txt','rb+')
print(f.write(b'0123455789abcdef'))
print(f.seek(5))
f.seek(-3, 2)
print(f.read(1))

import pickle,pprint

# 使用pickle模块将数据对象保存到文件
data1 = {'a': [1, 2.0, 3, 4+6j],
         'b': ('string', u'Unicode string'),
         'c': None}

selfref_list = [1, 2, 3]
selfref_list.append(selfref_list)

output = open('data.pkl', 'wb')

# Pickle dictionary using protocol 0.
pickle.dump(data1, output)

# Pickle the list using the highest protocol available.
pickle.dump(selfref_list, output, -1)

output.close()

#使用pickle模块从文件中重构python对象
pkl_file = open('data.pkl', 'rb')

data1 = pickle.load(pkl_file)
pprint.pprint(data1)

data2 = pickle.load(pkl_file)
pprint.pprint(data2)

pkl_file.close()


#! /usr/local/bin/python

from pathlib import Path
import pandas
import mysql.connector
from sqlalchemy import create_engine
import data.Database_Credentials as Database_Credentials

#Section: Create SQLAlchemy Engine
Database = 'Collections_Assessment_Warehouse_0.1'

Engine = create_engine(
    'mysql+mysqlconnector://' +
    Database_Credentials.Username + ':' +
    Database_Credentials.Password + '@' +
    Database_Credentials.Host + ':' + str(Database_Credentials.Post) + '/' +
    Database,
    echo=False # Should logging output be silenced?
)

#Section: Run SQL Query
SQL_Query = input("Enter the SQL statement to run (without newlines): ")

Query_Results = pandas.read_sql(
    sql=SQL_Query,
    con=Engine
)

#Section: Save Query Results to CSV
File_Name = input("Enter the name of the file (not including path or file extension info); don't use spaces or slashes. ")
File = str(Path('.', 'data', 'SQL_Output', File_Name + '.csv'))

Query_Results.to_csv(
    path_or_buf=File,
    index=False
)


from __future__ import annotations
from dataclasses import dataclass, field
from xsdata.models.datatype import XmlDate
from travelport.models.type_geo_political_area_type_1 import TypeGeoPoliticalAreaType1
from travelport.models.type_key_element_1 import TypeKeyElement1
from travelport.models.type_preference_purpose_1 import TypePreferencePurpose1

__NAMESPACE__ = "http://www.travelport.com/schema/sharedUprofile_v20_0"

@dataclass
class TypeBasePreferenceHistory1(TypeKeyElement1):
    """
    Base history element for the preference elements.

    Parameters
    ----------
    booking_start_date
    booking_end_date
    currency
    departure_geo_political_area_type
        Util: ReferenceDataRetrieveReq, TypeCodes  Airport, City Airport,
        City, Country, StateProvince and GeoPoliticalAreaType
    departure_geo_political_area_code
        Util: ReferenceDataRetrieveReq, TypeCodes  Airport, City Airport,
        City, Country, StateProvince and GeoPoliticalAreaType
    emphasis
    general_preference
    inclusive
        Indicates whether this preference is exclusive or inclusive (e.g.
        preference for not having a queen size bed vs preference to HAVE a
        queen size bed).
    loyalty_program_enrollment_ref
        A reference to a loyalty card element.
    other_loyalty_program_number
    payment_details_ref
        A reference to a payment details element list elsewhere.
    preference_payment_method
    purpose
        The purpose of the preference.
    priority_order
        Priority order associated with this Preference.
    supplier
    trip_approval
    """
    class Meta:
        name = "typeBasePreferenceHistory"

    booking_start_date: None | XmlDate = field(
        default=None,
        metadata={
            "name": "BookingStartDate",
            "type": "Attribute",
        }
    )
    booking_end_date: None | XmlDate = field(
        default=None,
        metadata={
            "name": "BookingEndDate",
            "type": "Attribute",
        }
    )
    currency: None | str = field(
        default=None,
        metadata={
            "name": "Currency",
            "type": "Attribute",
            "length": 3,
        }
    )
    departure_geo_political_area_type: None | TypeGeoPoliticalAreaType1 = field(
        default=None,
        metadata={
            "name": "DepartureGeoPoliticalAreaType",
            "type": "Attribute",
        }
    )
    departure_geo_political_area_code: None | str = field(
        default=None,
        metadata={
            "name": "DepartureGeoPoliticalAreaCode",
            "type": "Attribute",
            "max_length": 6,
        }
    )
    emphasis: None | bool = field(
        default=None,
        metadata={
            "name": "Emphasis",
            "type": "Attribute",
        }
    )
    general_preference: None | str = field(
        default=None,
        metadata={
            "name": "GeneralPreference",
            "type": "Attribute",
            "min_length": 1,
            "max_length": 255,
        }
    )
    inclusive: None | bool = field(
        default=None,
        metadata={
            "name": "Inclusive",
            "type": "Attribute",
        }
    )
    loyalty_program_enrollment_ref: None | str = field(
        default=None,
        metadata={
            "name": "LoyaltyProgramEnrollmentRef",
            "type": "Attribute",
        }
    )
    other_loyalty_program_number: None | str = field(
        default=None,
        metadata={
            "name": "OtherLoyaltyProgramNumber",
            "type": "Attribute",
            "max_length": 25,
        }
    )
    payment_details_ref: None | str = field(
        default=None,
        metadata={
            "name": "PaymentDetailsRef",
            "type": "Attribute",
        }
    )
    preference_payment_method: None | str = field(
        default=None,
        metadata={
            "name": "PreferencePaymentMethod",
            "type": "Attribute",
            "max_length": 6,
        }
    )
    purpose: None | TypePreferencePurpose1 = field(
        default=None,
        metadata={
            "name": "Purpose",
            "type": "Attribute",
        }
    )
    priority_order: None | int = field(
        default=None,
        metadata={
            "name": "PriorityOrder",
            "type": "Attribute",
            "min_inclusive": 1,
            "max_inclusive": 99,
        }
    )
    supplier: None | str = field(
        default=None,
        metadata={
            "name": "Supplier",
            "type": "Attribute",
            "max_length": 6,
        }
    )
    trip_approval: None | bool = field(
        default=None,
        metadata={
            "name": "TripApproval",
            "type": "Attribute",
        }
    )


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.index),
    url(r'^processreg$', views.processreg),
    url(r'^processlog$', views.processlog),
    url(r'^logout$', views.logout),
    url(r'^success$', views.success),
]


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re

class YYProto(object):
    def __init__(self):
        self.namespaces = []
        self.enums = []
        self.messages = []

class Enum(object):
    def __init__(self):
        pass

class Message(object):
    def __init__(self):
        self.name = ''
        self.fields = []
        self.id = 0

class Field(object):
    def __init__(self):
        self.type = ''
        self.name = ''

def parse_yyproto_file(filename):
    with open(filename) as f:
        return parse_yyproto(strip_comments(f.read()))

def parse_yyproto(yyproto):
    proto = YYProto()

    # parse namespaces
    namespaces, = re.findall(r'(namespace\s+[a-zA-Z_][a-zA-Z_0-9]*(?:.[a-zA-Z_][a-zA-Z_0-9]*)*\s*;)', yyproto)
    proto.namespaces = namespaces[0:-1].split()[1].split('.')

    # parse messages
    messages = re.findall(r'(message\s+[a-zA-Z_][a-zA-Z_0-9]*\s*{[^}]*};?)', yyproto)
    for message in messages:
        proto.messages.append(parse_message(message))

    return proto

def parse_message(message):
    msg = Message()

    name, = re.findall(r'(message\s+[a-zA-Z_][a-zA-Z_0-9]*\s*)', message)
    fields, = re.findall(r'({[^}]*})', message)

    msg.name = name.split()[1]
    fields = fields[1:-1].split(';')

    # message id
    if fields and re.match(r'enum\s+id\s*=\s*[1-9][0-9]*', strip_whitespace(fields[0])):
        msg.id = int(strip_whitespace(fields[0]).split('=')[1])
        fields = fields[1:]

    for field in fields:
        field = strip_whitespace(field)
        if field:
            msg.fields.append(parse_field(field))

    return msg

def parse_field(field):
    f = Field()
    f.type, f.name = field.split()
    return f

def strip_whitespace(s):
    return s.lstrip(' \t\n\r\f\v').rstrip(' \t\n\r\f\v')

def strip_comments(s):
    return re.sub('(//|#).*\n', '\n', s)

def main():
    import sys
    import jinja2

    if len(sys.argv) < 2:
        print('usage:yyprotoc example.yyproto')
        sys.exit(0)

    loader = jinja2.FileSystemLoader('templates')
    env = jinja2.Environment(loader=loader, trim_blocks=True)
    template = env.get_template('message.hpp.tpl')

    yyproto = parse_yyproto_file(sys.argv[1])
    with open(sys.argv[1] + '.hpp', 'w') as f:
        f.write(template.render(yyproto=yyproto))

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
from __future__ import division

def pico(l):
    posicao = 0
    for i in range (0,len(l)-1,1):
        if (l[i]>l[i+1]):
            posicao = i
            break

    cont = 0
    for i in range (posicao,len(l)-1,1):
        if (l[i]<=l[i+1]):
            cont = cont+1

    if (cont==0 and posicao!=0):
        return True
    else:
        return False

n = input ('Digite n:')
a = []

for i in range (0,n,1):
    a.append (input ('Digite um elemento da lista:'))

if pico(a):
    print ('S')
else:
    print ('N')


import os
import subprocess
import unittest

from sentry_check import (
    B001,
    B002,
    B003,
    B004,
    B005,
    B006,
    B007,
    B008,
    B009,
    B010,
    B101,
    B314,
    SentryCheck,
)

def path(*paths):
    return os.path.join(os.path.dirname(__file__), "cases", *paths)

class SentryCheckTestCase(unittest.TestCase):
    maxDiff = None

    def errors(self, *errors):
        return [SentryCheck.adapt_error(e) for e in errors]

    def test_b001(self):
        bbc = SentryCheck(filename=path("b001.py"))
        errors = list(bbc.run())
        self.assertEqual(errors, self.errors(B001(3, 0), B001(35, 4)))

    def test_b002(self):
        bbc = SentryCheck(filename=path("b002.py"))
        errors = list(bbc.run())
        self.assertEqual(errors, self.errors(B002(9, 8), B002(14, 11)))

    def test_b003(self):
        bbc = SentryCheck(filename=path("b003.py"))
        errors = list(bbc.run())
        self.assertEqual(errors, self.errors(B003(5, 0)))

    def test_b004(self):
        bbc = SentryCheck(filename=path("b004.py"))
        errors = list(bbc.run())
        self.assertEqual(errors, self.errors(B004(6, 7), B004(8, 7)))

    def test_b005(self):
        bbc = SentryCheck(filename=path("b005.py"))
        errors = list(bbc.run())
        self.assertEqual(
            errors,
            self.errors(
                B005(4, 0),
                B005(7, 0),
                B005(10, 0),
                B005(13, 0),
                B005(16, 0),
                B005(19, 0),
            ),
        )

    def test_b006_b008(self):
        bbc = SentryCheck(filename=path("b006_b008.py"))
        errors = list(bbc.run())
        self.assertEqual(
            errors,
            self.errors(
                B006(14, 24),
                B006(18, 29),
                B006(22, 19),
                B006(26, 19),
                B008(39, 38),
            ),
        )

    def test_b007(self):
        bbc = SentryCheck(filename=path("b007.py"))
        errors = list(bbc.run())
        self.assertEqual(
            errors,
            self.errors(
                B007(8, 4, vars=("i",)),
                B007(20, 12, vars=("k",)),
                B007(32, 4, vars=("i",)),
                B007(32, 12, vars=("k",)),
            ),
        )

    def test_b009_b010(self):
        bbc = SentryCheck(filename=path("b009_b010.py"))
        errors = list(bbc.run())
        self.assertEqual(errors, self.errors(B009(9, 0), B010(16, 0)))

    def test_b101(self):
        bbc = SentryCheck(filename=path("b101.py"))
        errors = list(bbc.run())
        self.assertEqual(errors, self.errors(B101(6, 0, vars=("assert_called_once",))))

    def test_b314(self):
        bbc = SentryCheck(filename=path("b314.py"))
        errors = list(bbc.run())
        self.assertEqual(errors, self.errors(B314(1, 0)))

    def test_b317(self):
        bbc = SentryCheck(filename=path("b317.py"))
        errors = list(bbc.run())
        assert errors == [
            (
                1,
                0,
                "B317: Use ``from sentry.utils import json`` instead.",
                SentryCheck,
            ),
            (
                2,
                0,
                "B317: Use ``from sentry.utils import json`` instead.",
                SentryCheck,
            ),
            (
                3,
                0,
                "B317: Use ``from sentry.utils import json`` instead.",
                SentryCheck,
            ),
            (
                4,
                0,
                "B317: Use ``from sentry.utils import json`` instead.",
                SentryCheck,
            ),
        ]

    def test_selfclean_sentry_check(self):
        stdout = subprocess.check_output(
            ["flake8", os.path.join(os.path.dirname(__file__), os.pardir, "src")]
        )
        self.assertEqual(stdout, b"")
        # self.assertEqual(proc.stderr, b"")

    def test_selfclean_test_sentry_check(self):
        stdout = subprocess.check_output(["flake8", __file__])
        self.assertEqual(stdout, b"")
        # self.assertEqual(proc.stderr, b'')

if __name__ == "__main__":
    unittest.main()


def task1(k):  # finds k number of sequence where xi = x(i-1) / 2x(i-2) - x(i-1)
    if k == 1: return 2.8
    j = 2
    x1, x2 = 2.8, -3.9
    for i in range(j, k):
        x1, x2 = x2, x2 / (2 * x1 - x2)
    return 'Number on the {} position of the sequence is {}'.format(k, x2)

def task2(x, eps = 0.000001):  # finds sum of row where S = 1 - (x / (1 * 3)) + (x^2 / (2 * 4)) - (x^3 / (3 * 5)) + ......
    if x ** 2 > 1: return 'Sum of given row with x = {} is equal to infinity'.format(x)
    check_eps = 1
    s = 1
    sign = False
    power = 1
    n1, n2 = 1, 3
    while check_eps > eps:
        if not sign:
            s -= x ** power / (n1 * n2)
        else:
            s += x ** power / (n1 * n2)
        sign = not sign
        power += 1
        n1, n2 = n1 + 2, n2 + 2
        check_eps = x ** power / (n1 * n2)
    return 'Sum of the given row with x = {} equal to {} '.format(x, s)

def task3(sequence):  # finds length of subsequence of the sequence where all elements are squares of int
    max_subseq = 0
    subseq = 0
    for i in sequence:
        if int(i ** 0.5) == float(i ** 0.5):
            subseq += 1
            if subseq > max_subseq:
                max_subseq = subseq
        else:
            subseq = 0
    return 'Input sequence = {}\n' \
           'Max length of the subsequence with squares of natural numbers is equal to {}'.format(sequence, max_subseq)

def task4(x1, x2, dx, eps = 0.000001):  # finds values of the Teilor row in range x1 -> x2 with step dx
    def find_func_value(x, eps):
        if x ** 2 <= 1:
            return 'infinity'
        power = 1
        coef = 1
        check_eps = 2
        s = 0
        while check_eps > eps:
            s += 1 / (coef * x ** power)
            coef += 2
            power += 2
            check_eps = 1 / (coef * x ** power)
        return str(s * 2)

    table = '\n|     x    |     y    |\n'
    while x1 < x2:
        if find_func_value(x1, eps) != 'infinity':
            table += '|{0:>10.3f}|{1:>10.3f}|\n'.format(x1, find_func_value(x1, eps))
        else:
            table += '|{0:>10.3f}| infinity |\n'.format(x1)
        x1 += dx
    return table

def fact(n):
    return fact(n - 1) if n != 1 else 1

def task5(x1, x2, dx, eps = 0.000001):
    def find_sin_teilor_function(x, eps):
        sign = False
        coef = 1
        sin = x
        check_eps = x
        while check_eps > eps:
            coef += 2
            check_eps = (x ** coef) / fact(coef)
            if not sign:
                sin -= (x ** coef) / fact(coef)
            else:
                sin += (x ** coef) / fact(coef)
            sign = not sign
        return sin

    table = '\n|     x    |     y    |\n'
    while x1 < x2:
        table += '|{0:>10.3f}|{1:>10.3f}|\n'.format(x1, find_sin_teilor_function(x1, eps) +
                                                    find_sin_teilor_function(2 * x1, eps))
        x1 += dx
    return table

def task6(x, eps = 0.000001):
    if not (0 < x <= 1):
        return 'Sum of the row with x = {} is equal to infinity'.format(x)
    s = 0
    k = 1
    element = x
    while element > eps:
        element = (((-1) ** (k + 1)) * (x ** (2 * k - 1))) / ((2 * k - 1) * fact(2 * k + 1))
        s += element
        k += 1
    return 'Sum of the row with x = {} is equal to {}'.format(x, s)

def task7(x, n):
    k = 1
    s = 0
    while k <= n:
        s += (((-1) ** k) * (x ** k)) / fact(fact(k) + 1)
        k += 1
    return 'Sum of the first n elements with x = {} of row is equal to {}'.format(x, s)

def task8(x, n):
    s = 0
    while n >= 1:
        s += (((-1) ** n) * (x ** n)) / fact(fact(n) + 1)
        n -= 1
    return 'Sum of the first n elements with x = {} of row is equal to {}'.format(x, s)


from typing import Sequence, Optional, NamedTuple
from operator import attrgetter
from semver import VersionInfo
from data_pipeline_api.metadata import Metadata, MetadataKey, is_superset

class MetadataRecord(NamedTuple):
    metadata: Metadata
    version: Optional[VersionInfo]

class MetadataStore:
    def __init__(self, metadata_sequence: Sequence[Metadata]):
        try:
            self._metadata_records = tuple(
                MetadataRecord(
                    metadata=metadata,
                    version=VersionInfo.parse(metadata[MetadataKey.version])
                    if MetadataKey.version in metadata
                    else None,
                )
                for metadata in metadata_sequence
            )
        except Exception as exception:
            raise ValueError("invalid metadata") from exception

    def find(self, metadata: Metadata) -> Optional[Metadata]:
        try:
            return max(
                filter(
                    lambda record: is_superset(record.metadata, metadata),
                    self._metadata_records,
                ),
                key=attrgetter("version"),
            ).metadata
        except ValueError:
            return None