lexer_compare_identifier_to_chars (const uint8_t *left_p, /**< left identifier */
                                   const uint8_t *right_p, /**< right identifier string */
                                   size_t size) /**< byte size of the two identifiers */
{
  uint8_t utf8_buf[6];

  do
  {
    if (*left_p == *right_p)
    {
      left_p++;
      right_p++;
      size--;
      continue;
    }

    size_t escape_size;

    if (*left_p == LIT_CHAR_BACKSLASH)
    {
      left_p += 2;
      lit_code_point_t code_point = lexer_unchecked_hex_to_character (&left_p);

      escape_size = lit_code_point_to_cesu8_bytes (utf8_buf, code_point);
    }
    else if (*left_p >= LIT_UTF8_4_BYTE_MARKER)
    {
      lit_four_byte_utf8_char_to_cesu8 (utf8_buf, left_p);
      escape_size = 3 * 2;
      left_p += 4;
    }
    else
    {
      return false;
    }

    size -= escape_size;

    uint8_t *utf8_p = utf8_buf;
    do
    {
      if (*right_p++ != *utf8_p++)
      {
        return false;
      }
    } while (--escape_size > 0);
  } while (size > 0);

  return true;
} /* lexer_compare_identifier_to_chars */


Fix class static block opening brace parsing (#4942)

The next character should not be consumed after finding the static block opening brace.
This patch fixes #4916.

JerryScript-DCO-1.0-Signed-off-by: Martin Negyokru negyokru@inf.u-szeged.hu

lexer_string_is_use_strict (parser_context_t *context_p) /**< context */
{
  JERRY_ASSERT (context_p->token.type == LEXER_LITERAL && context_p->token.lit_location.type == LEXER_STRING_LITERAL);

  return (context_p->token.lit_location.length == 10
          && !(context_p->token.lit_location.status_flags & LEXER_LIT_LOCATION_HAS_ESCAPE)
          && memcmp (context_p->token.lit_location.char_p, "use strict", 10) == 0);
} /* lexer_string_is_use_strict */


lexer_scan_private_identifier (parser_context_t *context_p) /**< context */
{
  context_p->token.keyword_type = LEXER_EOS;
  context_p->token.line = context_p->line;
  context_p->token.column = context_p->column;

  return (context_p->source_p < context_p->source_end_p && lexer_parse_identifier (context_p, LEXER_PARSE_NO_OPTS));
} /* lexer_scan_private_identifier */


lexer_construct_number_object (parser_context_t *context_p, /**< context */
                               bool is_expr, /**< expression is parsed */
                               bool is_negative_number) /**< sign is negative */
{
  parser_list_iterator_t literal_iterator;
  lexer_literal_t *literal_p;
  ecma_value_t lit_value;
  uint32_t literal_index = 0;
  prop_length_t length = context_p->token.lit_location.length;

#if JERRY_BUILTIN_BIGINT
  if (JERRY_LIKELY (context_p->token.extra_value != LEXER_NUMBER_BIGINT))
  {
#endif /* JERRY_BUILTIN_BIGINT */
    ecma_number_t num;
    uint32_t options = 0;

#if JERRY_ESNEXT
    options |= ECMA_CONVERSION_ALLOW_UNDERSCORE;
#endif /* JERRY_ESNEXT */

    if (context_p->token.extra_value == LEXER_NUMBER_OCTAL)
    {
      num = ecma_utf8_string_to_number_by_radix (context_p->token.lit_location.char_p, length, 8, options);
    }
    else
    {
      num = ecma_utf8_string_to_number (context_p->token.lit_location.char_p, length, options);
    }

    if (is_expr)
    {
      int32_t int_num = (int32_t) num;

      if (int_num == num && int_num <= CBC_PUSH_NUMBER_BYTE_RANGE_END && (int_num != 0 || !is_negative_number))
      {
        context_p->lit_object.index = (uint16_t) int_num;
        return true;
      }
    }

    if (is_negative_number)
    {
      num = -num;
    }

    lit_value = ecma_find_or_create_literal_number (num);
#if JERRY_BUILTIN_BIGINT
  }
  else
  {
    uint32_t options = (ECMA_BIGINT_PARSE_DISALLOW_SYNTAX_ERROR | ECMA_BIGINT_PARSE_DISALLOW_MEMORY_ERROR
                        | ECMA_BIGINT_PARSE_ALLOW_UNDERSCORE);

    if (is_negative_number)
    {
      options |= ECMA_BIGINT_PARSE_SET_NEGATIVE;
    }

    JERRY_ASSERT (length >= 2);
    lit_value =
      ecma_bigint_parse_string (context_p->token.lit_location.char_p, (lit_utf8_size_t) (length - 1), options);

    JERRY_ASSERT (lit_value != ECMA_VALUE_FALSE && !ECMA_IS_VALUE_ERROR (lit_value));

    if (lit_value == ECMA_VALUE_NULL)
    {
      parser_raise_error (context_p, PARSER_ERR_OUT_OF_MEMORY);
    }

    lit_value = ecma_find_or_create_literal_bigint (lit_value);
  }
#endif /* JERRY_BUILTIN_BIGINT */

  parser_list_iterator_init (&context_p->literal_pool, &literal_iterator);

  while ((literal_p = (lexer_literal_t *) parser_list_iterator_next (&literal_iterator)) != NULL)
  {
    if (literal_p->type == LEXER_NUMBER_LITERAL && literal_p->u.value == lit_value)
    {
      context_p->lit_object.literal_p = literal_p;
      context_p->lit_object.index = (uint16_t) literal_index;
      return false;
    }

    literal_index++;
  }

  JERRY_ASSERT (literal_index == context_p->literal_count);

  if (literal_index >= PARSER_MAXIMUM_NUMBER_OF_LITERALS)
  {
    parser_raise_error (context_p, PARSER_ERR_LITERAL_LIMIT_REACHED);
  }

  literal_p = (lexer_literal_t *) parser_list_append (context_p, &context_p->literal_pool);
  literal_p->u.value = lit_value;
  literal_p->prop.length = 0; /* Unused. */
  literal_p->type = LEXER_NUMBER_LITERAL;
  literal_p->status_flags = 0;

  context_p->lit_object.literal_p = literal_p;
  context_p->lit_object.index = (uint16_t) literal_index;

  context_p->literal_count++;
  return false;
} /* lexer_construct_number_object */


lexer_update_await_yield (parser_context_t *context_p, /**< context */
                          uint32_t status_flags) /**< parser status flags after restore */
{
  if (!(status_flags & PARSER_IS_STRICT))
  {
    if (status_flags & PARSER_IS_GENERATOR_FUNCTION)
    {
      if (context_p->token.type == LEXER_LITERAL && context_p->token.keyword_type == LEXER_KEYW_YIELD)
      {
        context_p->token.type = LEXER_KEYW_YIELD;
      }
    }
    else
    {
      if (context_p->token.type == LEXER_KEYW_YIELD)
      {
        JERRY_ASSERT (context_p->token.keyword_type == LEXER_KEYW_YIELD);
        context_p->token.type = LEXER_LITERAL;
      }
    }
  }

  if (!(context_p->global_status_flags & ECMA_PARSE_MODULE))
  {
    if (status_flags & PARSER_IS_ASYNC_FUNCTION)
    {
      if (context_p->token.type == LEXER_LITERAL && context_p->token.keyword_type == LEXER_KEYW_AWAIT)
      {
        context_p->token.type = LEXER_KEYW_AWAIT;
      }
    }
    else
    {
      if (context_p->token.type == LEXER_KEYW_AWAIT)
      {
        JERRY_ASSERT (context_p->token.keyword_type == LEXER_KEYW_AWAIT);
        context_p->token.type = LEXER_LITERAL;
      }
    }
  }
} /* lexer_update_await_yield */


lexer_consume_generator (parser_context_t *context_p) /**< context */
{
  if (!(context_p->token.flags & LEXER_NO_SKIP_SPACES))
  {
    lexer_skip_spaces (context_p);
    context_p->token.flags = (uint8_t) (context_p->token.flags | LEXER_NO_SKIP_SPACES);
  }

  if (context_p->source_p >= context_p->source_end_p || context_p->source_p[0] != LIT_CHAR_ASTERISK
      || (context_p->source_p + 1 < context_p->source_end_p
          && (context_p->source_p[1] == LIT_CHAR_EQUALS || context_p->source_p[1] == LIT_CHAR_ASTERISK)))
  {
    return false;
  }

  lexer_consume_next_character (context_p);
  context_p->token.type = LEXER_MULTIPLY;
  return true;
} /* lexer_consume_generator */


lexer_check_next_characters (parser_context_t *context_p, /**< context */
                             lit_utf8_byte_t character1, /**< first alternative character */
                             lit_utf8_byte_t character2) /**< second alternative character */
{
  if (!(context_p->token.flags & LEXER_NO_SKIP_SPACES))
  {
    lexer_skip_spaces (context_p);
    context_p->token.flags = (uint8_t) (context_p->token.flags | LEXER_NO_SKIP_SPACES);
  }

  return (context_p->source_p < context_p->source_end_p
          && (context_p->source_p[0] == (uint8_t) character1 || context_p->source_p[0] == (uint8_t) character2));
} /* lexer_check_next_characters */


lexer_skip_spaces (parser_context_t *context_p) /**< context */
{
  skip_mode_t mode = LEXER_SKIP_SPACES;
  const uint8_t *source_end_p = context_p->source_end_p;

  if (context_p->token.flags & LEXER_NO_SKIP_SPACES)
  {
    context_p->token.flags &= (uint8_t) ~LEXER_NO_SKIP_SPACES;
    return;
  }

  context_p->token.flags = 0;

  while (true)
  {
    if (context_p->source_p >= source_end_p)
    {
      if (mode == LEXER_SKIP_MULTI_LINE_COMMENT)
      {
        parser_raise_error (context_p, PARSER_ERR_UNTERMINATED_MULTILINE_COMMENT);
      }
      return;
    }

    switch (context_p->source_p[0])
    {
      case LIT_CHAR_CR:
      {
        if (context_p->source_p + 1 < source_end_p && context_p->source_p[1] == LIT_CHAR_LF)
        {
          context_p->source_p++;
        }
        /* FALLTHRU */
      }

      case LIT_CHAR_LF:
      {
        context_p->line++;
        context_p->column = 0;
        context_p->token.flags = LEXER_WAS_NEWLINE;

        if (mode == LEXER_SKIP_SINGLE_LINE_COMMENT)
        {
          mode = LEXER_SKIP_SPACES;
        }
        /* FALLTHRU */
      }

      case LIT_CHAR_VTAB:
      case LIT_CHAR_FF:
      case LIT_CHAR_SP:
      {
        context_p->source_p++;
        context_p->column++;
        continue;
      }

      case LIT_CHAR_TAB:
      {
        context_p->column = align_column_to_tab (context_p->column);
        context_p->source_p++;
        continue;
      }

      case LIT_CHAR_SLASH:
      {
        if (mode == LEXER_SKIP_SPACES && context_p->source_p + 1 < source_end_p)
        {
          if (context_p->source_p[1] == LIT_CHAR_SLASH)
          {
            mode = LEXER_SKIP_SINGLE_LINE_COMMENT;
          }
          else if (context_p->source_p[1] == LIT_CHAR_ASTERISK)
          {
            mode = LEXER_SKIP_MULTI_LINE_COMMENT;
            context_p->token.line = context_p->line;
            context_p->token.column = context_p->column;
          }

          if (mode != LEXER_SKIP_SPACES)
          {
            context_p->source_p += 2;
            PARSER_PLUS_EQUAL_LC (context_p->column, 2);
            continue;
          }
        }
        break;
      }

      case LIT_CHAR_ASTERISK:
      {
        if (mode == LEXER_SKIP_MULTI_LINE_COMMENT && context_p->source_p + 1 < source_end_p
            && context_p->source_p[1] == LIT_CHAR_SLASH)
        {
          mode = LEXER_SKIP_SPACES;
          context_p->source_p += 2;
          PARSER_PLUS_EQUAL_LC (context_p->column, 2);
          continue;
        }
        break;
      }

      case 0xc2:
      {
        if (context_p->source_p + 1 < source_end_p && context_p->source_p[1] == 0xa0)
        {
          /* Codepoint \u00A0 */
          context_p->source_p += 2;
          context_p->column++;
          continue;
        }
        break;
      }

      case LEXER_NEWLINE_LS_PS_BYTE_1:
      {
        JERRY_ASSERT (context_p->source_p + 2 < source_end_p);
        if (LEXER_NEWLINE_LS_PS_BYTE_23 (context_p->source_p))
        {
          /* Codepoint \u2028 and \u2029 */
          context_p->source_p += 3;
          context_p->line++;
          context_p->column = 1;
          context_p->token.flags = LEXER_WAS_NEWLINE;

          if (mode == LEXER_SKIP_SINGLE_LINE_COMMENT)
          {
            mode = LEXER_SKIP_SPACES;
          }
          continue;
        }
        break;
      }

      case 0xef:
      {
        if (context_p->source_p + 2 < source_end_p && context_p->source_p[1] == 0xbb && context_p->source_p[2] == 0xbf)
        {
          /* Codepoint \uFEFF */
          context_p->source_p += 3;
          context_p->column++;
          continue;
        }
        break;
      }

      default:
      {
        break;
      }
    }

    if (mode == LEXER_SKIP_SPACES)
    {
      return;
    }

    context_p->source_p++;

    if (context_p->source_p < source_end_p && !IS_UTF8_INTERMEDIATE_OCTET (context_p->source_p[0]))
    {
      context_p->column++;
    }
  }
} /* lexer_skip_spaces */


lexer_skip_empty_statements (parser_context_t *context_p) /**< context */
{
  lexer_skip_spaces (context_p);

  while (context_p->source_p < context_p->source_end_p && *context_p->source_p == LIT_CHAR_SEMICOLON)
  {
    lexer_consume_next_character (context_p);
    lexer_skip_spaces (context_p);
  }

  context_p->token.flags = (uint8_t) (context_p->token.flags | LEXER_NO_SKIP_SPACES);
} /* lexer_skip_empty_statements */


lexer_parse_string (parser_context_t *context_p, /**< context */
                    lexer_string_options_t opts) /**< options */
{
#if JERRY_ESNEXT
  int32_t raw_length_adjust = 0;
#else /* JERRY_ESNEXT */
  JERRY_UNUSED (opts);
#endif /* JERRY_ESNEXT */

  uint8_t str_end_character = context_p->source_p[0];
  const uint8_t *source_p = context_p->source_p + 1;
  const uint8_t *string_start_p = source_p;
  const uint8_t *source_end_p = context_p->source_end_p;
  parser_line_counter_t line = context_p->line;
  parser_line_counter_t column = (parser_line_counter_t) (context_p->column + 1);
  parser_line_counter_t original_line = line;
  parser_line_counter_t original_column = column;
  size_t length = 0;
  lexer_lit_location_flags_t status_flags = LEXER_LIT_LOCATION_IS_ASCII;

#if JERRY_ESNEXT
  if (str_end_character == LIT_CHAR_RIGHT_BRACE)
  {
    str_end_character = LIT_CHAR_GRAVE_ACCENT;
  }
#endif /* JERRY_ESNEXT */

  while (true)
  {
    if (source_p >= source_end_p)
    {
      context_p->token.line = original_line;
      context_p->token.column = (parser_line_counter_t) (original_column - 1);
      parser_raise_error (context_p, PARSER_ERR_UNTERMINATED_STRING);
    }

    if (*source_p == str_end_character)
    {
      break;
    }

    if (*source_p == LIT_CHAR_BACKSLASH)
    {
      source_p++;
      column++;
      if (source_p >= source_end_p)
      {
        /* Will throw an unterminated string error. */
        continue;
      }

      status_flags = LEXER_LIT_LOCATION_HAS_ESCAPE;

      /* Newline is ignored. */
      if (*source_p == LIT_CHAR_CR)
      {
        source_p++;
        if (source_p < source_end_p && *source_p == LIT_CHAR_LF)
        {
#if JERRY_ESNEXT
          raw_length_adjust--;
#endif /* JERRY_ESNEXT */
          source_p++;
        }

        line++;
        column = 1;
        continue;
      }
      else if (*source_p == LIT_CHAR_LF)
      {
        source_p++;
        line++;
        column = 1;
        continue;
      }
      else if (*source_p == LEXER_NEWLINE_LS_PS_BYTE_1 && LEXER_NEWLINE_LS_PS_BYTE_23 (source_p))
      {
        source_p += 3;
        line++;
        column = 1;
        continue;
      }

#if JERRY_ESNEXT
      if (opts & LEXER_STRING_RAW)
      {
        if ((*source_p == LIT_CHAR_GRAVE_ACCENT) || (*source_p == LIT_CHAR_BACKSLASH))
        {
          source_p++;
          column++;
          length++;
        }
        continue;
      }
#endif /* JERRY_ESNEXT */

      if (*source_p == LIT_CHAR_0 && source_p + 1 < source_end_p
          && (*(source_p + 1) < LIT_CHAR_0 || *(source_p + 1) > LIT_CHAR_9))
      {
        source_p++;
        column++;
        length++;
        continue;
      }

      /* Except \x, \u, and octal numbers, everything is
       * converted to a character which has the same byte length. */
      if (*source_p >= LIT_CHAR_0 && *source_p <= LIT_CHAR_3)
      {
#if JERRY_ESNEXT
        if (str_end_character == LIT_CHAR_GRAVE_ACCENT)
        {
          parser_raise_error (context_p, PARSER_ERR_TEMPLATE_STR_OCTAL_ESCAPE);
        }
#endif /* JERRY_ESNEXT */

        if (context_p->status_flags & PARSER_IS_STRICT)
        {
          parser_raise_error (context_p, PARSER_ERR_OCTAL_ESCAPE_NOT_ALLOWED);
        }

        source_p++;
        column++;

        if (source_p < source_end_p && *source_p >= LIT_CHAR_0 && *source_p <= LIT_CHAR_7)
        {
          source_p++;
          column++;

          if (source_p < source_end_p && *source_p >= LIT_CHAR_0 && *source_p <= LIT_CHAR_7)
          {
            /* Numbers >= 0x200 (0x80) requires
             * two bytes for encoding in UTF-8. */
            if (source_p[-2] >= LIT_CHAR_2)
            {
              length++;
            }

            source_p++;
            column++;
          }
        }

        length++;
        continue;
      }

      if (*source_p >= LIT_CHAR_4 && *source_p <= LIT_CHAR_7)
      {
        if (context_p->status_flags & PARSER_IS_STRICT)
        {
          parser_raise_error (context_p, PARSER_ERR_OCTAL_ESCAPE_NOT_ALLOWED);
        }

        source_p++;
        column++;

        if (source_p < source_end_p && *source_p >= LIT_CHAR_0 && *source_p <= LIT_CHAR_7)
        {
          source_p++;
          column++;
        }

        /* The maximum number is 0x4d so the UTF-8
         * representation is always one byte. */
        length++;
        continue;
      }

      if (*source_p == LIT_CHAR_LOWERCASE_X || *source_p == LIT_CHAR_LOWERCASE_U)
      {
        uint32_t escape_length = (*source_p == LIT_CHAR_LOWERCASE_X) ? 3 : 5;
        lit_code_point_t code_point = UINT32_MAX;

#if JERRY_ESNEXT
        if (source_p + 4 <= source_end_p && source_p[0] == LIT_CHAR_LOWERCASE_U && source_p[1] == LIT_CHAR_LEFT_BRACE)
        {
          code_point = lexer_hex_in_braces_to_code_point (source_p + 2, source_end_p, &escape_length);
          escape_length--;
        }
        else
        {
#endif /* JERRY_ESNEXT */
          if (source_p + escape_length <= source_end_p)
          {
            code_point = lexer_hex_to_code_point (source_p + 1, escape_length - 1);
          }
#if JERRY_ESNEXT
        }
#endif /* JERRY_ESNEXT */

        if (code_point == UINT32_MAX)
        {
          context_p->token.line = line;
          context_p->token.column = (parser_line_counter_t) (column - 1);
          parser_raise_error (context_p, PARSER_ERR_INVALID_UNICODE_ESCAPE_SEQUENCE);
        }

        length += lit_code_point_get_cesu8_length (code_point);

        source_p += escape_length;
        PARSER_PLUS_EQUAL_LC (column, escape_length);
        continue;
      }
    }
#if JERRY_ESNEXT
    else if (str_end_character == LIT_CHAR_GRAVE_ACCENT && source_p[0] == LIT_CHAR_DOLLAR_SIGN
             && source_p + 1 < source_end_p && source_p[1] == LIT_CHAR_LEFT_BRACE)
    {
      raw_length_adjust--;
      source_p++;
      break;
    }
#endif /* JERRY_ESNEXT */

    if (*source_p >= LIT_UTF8_4_BYTE_MARKER)
    {
      /* Processing 4 byte unicode sequence (even if it is
       * after a backslash). Always converted to two 3 byte
       * long sequence. */
      length += 2 * 3;
      status_flags = LEXER_LIT_LOCATION_HAS_ESCAPE;
      source_p += 4;
#if JERRY_ESNEXT
      raw_length_adjust += 2;
#endif /* JERRY_ESNEXT */
      column++;
#if JERRY_FUNCTION_TO_STRING
      context_p->global_status_flags |= ECMA_PARSE_INTERNAL_HAS_4_BYTE_MARKER;
#endif /* JERRY_FUNCTION_TO_STRING */
      continue;
    }
    else if (*source_p == LIT_CHAR_TAB)
    {
      column = align_column_to_tab (column);
      /* Subtract -1 because column is increased below. */
      column--;
    }
#if JERRY_ESNEXT
    else if (*source_p == LEXER_NEWLINE_LS_PS_BYTE_1 && LEXER_NEWLINE_LS_PS_BYTE_23 (source_p))
    {
      source_p += 3;
      length += 3;
      line++;
      column = 1;
      continue;
    }
    else if (str_end_character == LIT_CHAR_GRAVE_ACCENT)
    {
      /* Newline (without backslash) is part of the string.
         Note: ECMAScript v6, 11.8.6.1 <CR> or <CR><LF> are both normalized to <LF> */
      if (*source_p == LIT_CHAR_CR)
      {
        status_flags = LEXER_LIT_LOCATION_HAS_ESCAPE;
        source_p++;
        length++;
        if (source_p < source_end_p && *source_p == LIT_CHAR_LF)
        {
          source_p++;
          raw_length_adjust--;
        }
        line++;
        column = 1;
        continue;
      }
      else if (*source_p == LIT_CHAR_LF)
      {
        source_p++;
        length++;
        line++;
        column = 1;
        continue;
      }
    }
#endif /* JERRY_ESNEXT */
    else if (*source_p == LIT_CHAR_CR
#if !JERRY_ESNEXT
             || (*source_p == LEXER_NEWLINE_LS_PS_BYTE_1 && LEXER_NEWLINE_LS_PS_BYTE_23 (source_p))
#endif /* !JERRY_ESNEXT */
             || *source_p == LIT_CHAR_LF)
    {
      context_p->token.line = line;
      context_p->token.column = column;
      parser_raise_error (context_p, PARSER_ERR_NEWLINE_NOT_ALLOWED);
    }

    source_p++;
    column++;
    length++;

    while (source_p < source_end_p && IS_UTF8_INTERMEDIATE_OCTET (*source_p))
    {
      source_p++;
      length++;
    }
  }

#if JERRY_ESNEXT
  if (opts & LEXER_STRING_RAW)
  {
    length = (size_t) ((source_p - string_start_p) + raw_length_adjust);
  }
#endif /* JERRY_ESNEXT */

  if (length > PARSER_MAXIMUM_STRING_LENGTH)
  {
    parser_raise_error (context_p, PARSER_ERR_STRING_TOO_LONG);
  }

#if JERRY_ESNEXT
  context_p->token.type = ((str_end_character != LIT_CHAR_GRAVE_ACCENT) ? LEXER_LITERAL : LEXER_TEMPLATE_LITERAL);
#else /* !JERRY_ESNEXT */
  context_p->token.type = LEXER_LITERAL;
#endif /* JERRY_ESNEXT */

  /* Fill literal data. */
  context_p->token.lit_location.char_p = string_start_p;
  context_p->token.lit_location.length = (prop_length_t) length;
  context_p->token.lit_location.type = LEXER_STRING_LITERAL;
  context_p->token.lit_location.status_flags = (uint8_t) status_flags;

  context_p->source_p = source_p + 1;
  context_p->line = line;
  context_p->column = (parser_line_counter_t) (column + 1);
} /* lexer_parse_string */


lexer_token_is_async (parser_context_t *context_p) /**< context */
{
  JERRY_ASSERT (context_p->token.type == LEXER_LITERAL || context_p->token.type == LEXER_TEMPLATE_LITERAL);

  return (context_p->token.keyword_type == LEXER_KEYW_ASYNC
          && !(context_p->token.lit_location.status_flags & LEXER_LIT_LOCATION_HAS_ESCAPE));
} /* lexer_token_is_async */


lexer_convert_ident_to_cesu8 (uint8_t *destination_p, /**< destination string */
                              const uint8_t *source_p, /**< source string */
                              prop_length_t length) /**< length of destination string */
{
  const uint8_t *destination_end_p = destination_p + length;

  JERRY_ASSERT (length <= PARSER_MAXIMUM_IDENT_LENGTH);

  do
  {
    if (*source_p == LIT_CHAR_BACKSLASH)
    {
      source_p += 2;
      destination_p += lit_code_point_to_cesu8_bytes (destination_p, lexer_unchecked_hex_to_character (&source_p));
      continue;
    }

#if JERRY_ESNEXT
    if (*source_p >= LIT_UTF8_4_BYTE_MARKER)
    {
      lit_four_byte_utf8_char_to_cesu8 (destination_p, source_p);

      destination_p += 6;
      source_p += 4;
      continue;
    }
#endif /* JERRY_ESNEXT */

    *destination_p++ = *source_p++;
  } while (destination_p < destination_end_p);
} /* lexer_convert_ident_to_cesu8 */


lexer_construct_literal_object (parser_context_t *context_p, /**< context */
                                const lexer_lit_location_t *lit_location_p, /**< literal location */
                                uint8_t literal_type) /**< final literal type */
{
  uint8_t local_byte_array[LEXER_MAX_LITERAL_LOCAL_BUFFER_SIZE];

  const uint8_t *char_p =
    lexer_convert_literal_to_chars (context_p, lit_location_p, local_byte_array, LEXER_STRING_NO_OPTS);

  size_t length = lit_location_p->length;
  parser_list_iterator_t literal_iterator;
  lexer_literal_t *literal_p;
  uint32_t literal_index = 0;
  bool search_scope_stack = (literal_type == LEXER_IDENT_LITERAL);

  if (JERRY_UNLIKELY (literal_type == LEXER_NEW_IDENT_LITERAL))
  {
    literal_type = LEXER_IDENT_LITERAL;
  }

  JERRY_ASSERT (literal_type == LEXER_IDENT_LITERAL || literal_type == LEXER_STRING_LITERAL);

  JERRY_ASSERT (literal_type != LEXER_IDENT_LITERAL || length <= PARSER_MAXIMUM_IDENT_LENGTH);
  JERRY_ASSERT (literal_type != LEXER_STRING_LITERAL || length <= PARSER_MAXIMUM_STRING_LENGTH);

  parser_list_iterator_init (&context_p->literal_pool, &literal_iterator);

  while ((literal_p = (lexer_literal_t *) parser_list_iterator_next (&literal_iterator)) != NULL)
  {
    if (literal_p->type == literal_type && literal_p->prop.length == length
        && memcmp (literal_p->u.char_p, char_p, length) == 0)
    {
      context_p->lit_object.literal_p = literal_p;
      context_p->lit_object.index = (uint16_t) literal_index;

      parser_free_allocated_buffer (context_p);

      if (search_scope_stack)
      {
        parser_scope_stack_t *scope_stack_start_p = context_p->scope_stack_p;
        parser_scope_stack_t *scope_stack_p = scope_stack_start_p + context_p->scope_stack_top;

        while (scope_stack_p > scope_stack_start_p)
        {
          scope_stack_p--;

          if (scope_stack_p->map_from == literal_index)
          {
            JERRY_ASSERT (scanner_decode_map_to (scope_stack_p) >= PARSER_REGISTER_START
                          || (literal_p->status_flags & LEXER_FLAG_USED));
            context_p->lit_object.index = scanner_decode_map_to (scope_stack_p);
            return;
          }
        }

        literal_p->status_flags |= LEXER_FLAG_USED;
      }
      return;
    }

    literal_index++;
  }

  JERRY_ASSERT (literal_index == context_p->literal_count);

  if (literal_index >= PARSER_MAXIMUM_NUMBER_OF_LITERALS)
  {
    parser_raise_error (context_p, PARSER_ERR_LITERAL_LIMIT_REACHED);
  }

  literal_p = (lexer_literal_t *) parser_list_append (context_p, &context_p->literal_pool);
  literal_p->prop.length = (prop_length_t) length;
  literal_p->type = literal_type;

  uint8_t status_flags = LEXER_FLAG_SOURCE_PTR;

  if (length > 0 && char_p == local_byte_array)
  {
    literal_p->u.char_p = (uint8_t *) jmem_heap_alloc_block (length);
    memcpy ((uint8_t *) literal_p->u.char_p, char_p, length);
    status_flags = 0;
  }
  else
  {
    literal_p->u.char_p = char_p;

    /* Buffer is taken over when a new literal is constructed. */
    if (context_p->u.allocated_buffer_p != NULL)
    {
      JERRY_ASSERT (char_p == context_p->u.allocated_buffer_p);

      context_p->u.allocated_buffer_p = NULL;
      status_flags = 0;
    }
  }

  if (search_scope_stack)
  {
    status_flags |= LEXER_FLAG_USED;
  }

  if (lit_location_p->status_flags & LEXER_LIT_LOCATION_IS_ASCII)
  {
    literal_p->status_flags |= LEXER_FLAG_ASCII;
  }

  literal_p->status_flags = status_flags;

  context_p->lit_object.literal_p = literal_p;
  context_p->lit_object.index = (uint16_t) literal_index;
  context_p->literal_count++;

  JERRY_ASSERT (context_p->u.allocated_buffer_p == NULL);
} /* lexer_construct_literal_object */


lexer_check_numbers (parser_context_t *context_p, /**< context */
                     const uint8_t **source_p, /**< source_pointer */
                     const uint8_t *source_end_p, /**< end of the source */
                     const ecma_char_t digit_max, /**< maximum of the number range */
                     const bool is_legacy) /**< is legacy octal number  */
{
#if !JERRY_ESNEXT
  JERRY_UNUSED (context_p);
  JERRY_UNUSED (is_legacy);
#endif /* !JERRY_ESNEXT */
  while (true)
  {
    while (*source_p < source_end_p && *source_p[0] >= LIT_CHAR_0 && *source_p[0] <= digit_max)
    {
      *source_p += 1;
    }
#if JERRY_ESNEXT
    if (*source_p != source_end_p && *source_p[0] == LIT_CHAR_UNDERSCORE)
    {
      *source_p += 1;
      if (is_legacy || *source_p == source_end_p || *source_p[0] == LIT_CHAR_UNDERSCORE || *source_p[0] > digit_max
          || *source_p[0] < LIT_CHAR_0)
      {
        parser_raise_error (context_p, PARSER_ERR_INVALID_UNDERSCORE_IN_NUMBER);
      }
      continue;
    }
#endif /* JERRY_ESNEXT */

    break;
  }
} /* lexer_check_numbers */


lexer_check_arrow_param (parser_context_t *context_p) /**< context */
{
  JERRY_ASSERT (context_p->token.flags & LEXER_NO_SKIP_SPACES);

  if (context_p->source_p >= context_p->source_end_p)
  {
    return false;
  }

  if (context_p->source_p[0] == LIT_CHAR_COMMA)
  {
    return true;
  }

  if (context_p->source_p[0] != LIT_CHAR_EQUALS)
  {
    return false;
  }

  return (context_p->source_p + 1 >= context_p->source_end_p || context_p->source_p[1] != LIT_CHAR_EQUALS);
} /* lexer_check_arrow_param */


lexer_parse_number (parser_context_t *context_p) /**< context */
{
  const uint8_t *source_p = context_p->source_p;
  const uint8_t *source_end_p = context_p->source_end_p;
  bool can_be_float = false;
#if JERRY_BUILTIN_BIGINT
  bool can_be_bigint = true;
#endif /* JERRY_BUILTIN_BIGINT */
  size_t length;

  context_p->token.type = LEXER_LITERAL;
  context_p->token.extra_value = LEXER_NUMBER_DECIMAL;
  context_p->token.lit_location.char_p = source_p;
  context_p->token.lit_location.type = LEXER_NUMBER_LITERAL;
  context_p->token.lit_location.status_flags = LEXER_LIT_LOCATION_IS_ASCII;

  if (source_p[0] == LIT_CHAR_0 && source_p + 1 < source_end_p)
  {
#if JERRY_ESNEXT
    if (source_p[1] == LIT_CHAR_UNDERSCORE)
    {
      parser_raise_error (context_p, PARSER_ERR_INVALID_UNDERSCORE_IN_NUMBER);
    }
#endif /* JERRY_ESNEXT */
    if (LEXER_TO_ASCII_LOWERCASE (source_p[1]) == LIT_CHAR_LOWERCASE_X)
    {
      context_p->token.extra_value = LEXER_NUMBER_HEXADECIMAL;
      source_p += 2;

      if (source_p >= source_end_p || !lit_char_is_hex_digit (source_p[0]))
      {
        parser_raise_error (context_p, PARSER_ERR_INVALID_HEX_DIGIT);
      }

      do
      {
        source_p++;
#if JERRY_ESNEXT
        if (source_p < source_end_p && source_p[0] == LIT_CHAR_UNDERSCORE)
        {
          source_p++;
          if (source_p == source_end_p || !lit_char_is_hex_digit (source_p[0]))
          {
            parser_raise_error (context_p, PARSER_ERR_INVALID_UNDERSCORE_IN_NUMBER);
          }
        }
#endif /* JERRY_ESNEXT */
      } while (source_p < source_end_p && lit_char_is_hex_digit (source_p[0]));
    }
#if JERRY_ESNEXT
    else if (LEXER_TO_ASCII_LOWERCASE (source_p[1]) == LIT_CHAR_LOWERCASE_O)
    {
      context_p->token.extra_value = LEXER_NUMBER_OCTAL;
      source_p += 2;

      if (source_p >= source_end_p || !lit_char_is_octal_digit (source_p[0]))
      {
        parser_raise_error (context_p, PARSER_ERR_INVALID_OCTAL_DIGIT);
      }

      lexer_check_numbers (context_p, &source_p, source_end_p, LIT_CHAR_7, false);
    }
#endif /* JERRY_ESNEXT */
    else if (source_p[1] >= LIT_CHAR_0 && source_p[1] <= LIT_CHAR_9)
    {
      context_p->token.extra_value = LEXER_NUMBER_OCTAL;
#if JERRY_BUILTIN_BIGINT
      can_be_bigint = false;
#endif /* JERRY_BUILTIN_BIGINT */

      if (context_p->status_flags & PARSER_IS_STRICT)
      {
        parser_raise_error (context_p, PARSER_ERR_OCTAL_NUMBER_NOT_ALLOWED);
      }

      lexer_check_numbers (context_p, &source_p, source_end_p, LIT_CHAR_7, true);

      if (source_p < source_end_p && source_p[0] >= LIT_CHAR_8 && source_p[0] <= LIT_CHAR_9)
      {
#if JERRY_ESNEXT
        lexer_check_numbers (context_p, &source_p, source_end_p, LIT_CHAR_9, true);
        context_p->token.extra_value = LEXER_NUMBER_DECIMAL;
#else /* !JERRY_ESNEXT */
        parser_raise_error (context_p, PARSER_ERR_INVALID_NUMBER);
#endif /* JERRY_ESNEXT */
      }
    }
#if JERRY_ESNEXT
    else if (LEXER_TO_ASCII_LOWERCASE (source_p[1]) == LIT_CHAR_LOWERCASE_B)
    {
      context_p->token.extra_value = LEXER_NUMBER_BINARY;
      source_p += 2;

      if (source_p >= source_end_p || !lit_char_is_binary_digit (source_p[0]))
      {
        parser_raise_error (context_p, PARSER_ERR_INVALID_BIN_DIGIT);
      }

      do
      {
        source_p++;
        if (source_p < source_end_p && source_p[0] == LIT_CHAR_UNDERSCORE)
        {
          source_p++;
          if (source_p == source_end_p || source_p[0] > LIT_CHAR_9 || source_p[0] < LIT_CHAR_0)
          {
            parser_raise_error (context_p, PARSER_ERR_INVALID_UNDERSCORE_IN_NUMBER);
          }
        }
      } while (source_p < source_end_p && lit_char_is_binary_digit (source_p[0]));
    }
#endif /* JERRY_ESNEXT */
    else
    {
      can_be_float = true;
      source_p++;
    }
  }
  else
  {
    lexer_check_numbers (context_p, &source_p, source_end_p, LIT_CHAR_9, false);
    can_be_float = true;
  }

  if (can_be_float)
  {
    if (source_p < source_end_p && source_p[0] == LIT_CHAR_DOT)
    {
      source_p++;
#if JERRY_BUILTIN_BIGINT
      can_be_bigint = false;
#endif /* JERRY_BUILTIN_BIGINT */

#if JERRY_ESNEXT
      if (source_p < source_end_p && source_p[0] == LIT_CHAR_UNDERSCORE)
      {
        parser_raise_error (context_p, PARSER_ERR_INVALID_UNDERSCORE_IN_NUMBER);
      }
#endif /* JERRY_ESNEXT */
      lexer_check_numbers (context_p, &source_p, source_end_p, LIT_CHAR_9, false);
    }

    if (source_p < source_end_p && LEXER_TO_ASCII_LOWERCASE (source_p[0]) == LIT_CHAR_LOWERCASE_E)
    {
      source_p++;
#if JERRY_BUILTIN_BIGINT
      can_be_bigint = false;
#endif /* JERRY_BUILTIN_BIGINT */

      if (source_p < source_end_p && (source_p[0] == LIT_CHAR_PLUS || source_p[0] == LIT_CHAR_MINUS))
      {
        source_p++;
      }

      if (source_p >= source_end_p || source_p[0] < LIT_CHAR_0 || source_p[0] > LIT_CHAR_9)
      {
        parser_raise_error (context_p, PARSER_ERR_MISSING_EXPONENT);
      }

      lexer_check_numbers (context_p, &source_p, source_end_p, LIT_CHAR_9, false);
    }
  }

#if JERRY_BUILTIN_BIGINT
  if (source_p < source_end_p && source_p[0] == LIT_CHAR_LOWERCASE_N)
  {
    if (!can_be_bigint)
    {
      parser_raise_error (context_p, PARSER_ERR_INVALID_BIGINT);
    }
    context_p->token.extra_value = LEXER_NUMBER_BIGINT;
    source_p++;
  }
#endif /* JERRY_BUILTIN_BIGINT */

  length = (size_t) (source_p - context_p->source_p);
  if (length > PARSER_MAXIMUM_STRING_LENGTH)
  {
    parser_raise_error (context_p, PARSER_ERR_NUMBER_TOO_LONG);
  }

  context_p->token.lit_location.length = (prop_length_t) length;
  PARSER_PLUS_EQUAL_LC (context_p->column, length);
  context_p->source_p = source_p;

  if (source_p < source_end_p && lexer_parse_identifier (context_p, LEXER_PARSE_CHECK_START_AND_RETURN))
  {
    parser_raise_error (context_p, PARSER_ERR_IDENTIFIER_AFTER_NUMBER);
  }
} /* lexer_parse_number */


lexer_check_arrow (parser_context_t *context_p) /**< context */
{
  if (!(context_p->token.flags & LEXER_NO_SKIP_SPACES))
  {
    lexer_skip_spaces (context_p);
    context_p->token.flags = (uint8_t) (context_p->token.flags | LEXER_NO_SKIP_SPACES);
  }

  return (!(context_p->token.flags & LEXER_WAS_NEWLINE) && context_p->source_p + 2 <= context_p->source_end_p
          && context_p->source_p[0] == (uint8_t) LIT_CHAR_EQUALS
          && context_p->source_p[1] == (uint8_t) LIT_CHAR_GREATER_THAN);
} /* lexer_check_arrow */


lexer_consume_assign (parser_context_t *context_p) /**< context */
{
  if (!(context_p->token.flags & LEXER_NO_SKIP_SPACES))
  {
    lexer_skip_spaces (context_p);
    context_p->token.flags = (uint8_t) (context_p->token.flags | LEXER_NO_SKIP_SPACES);
  }

  if (context_p->source_p >= context_p->source_end_p || context_p->source_p[0] != LIT_CHAR_EQUALS
      || (context_p->source_p + 1 < context_p->source_end_p
          && (context_p->source_p[1] == LIT_CHAR_EQUALS || context_p->source_p[1] == LIT_CHAR_GREATER_THAN)))
  {
    return false;
  }

  lexer_consume_next_character (context_p);
  context_p->token.type = LEXER_ASSIGN;
  return true;
} /* lexer_consume_assign */


lexer_construct_regexp_object (parser_context_t *context_p, /**< context */
                               bool parse_only) /**< parse only */
{
#if JERRY_BUILTIN_REGEXP
  const uint8_t *source_p = context_p->source_p;
  const uint8_t *regex_start_p = context_p->source_p;
  const uint8_t *regex_end_p = regex_start_p;
  const uint8_t *source_end_p = context_p->source_end_p;
  parser_line_counter_t column = context_p->column;
  bool in_class = false;
  uint16_t current_flags;
  lit_utf8_size_t length;

  JERRY_ASSERT (context_p->token.type == LEXER_DIVIDE || context_p->token.type == LEXER_ASSIGN_DIVIDE);

  if (context_p->token.type == LEXER_ASSIGN_DIVIDE)
  {
    regex_start_p--;
  }

  while (true)
  {
    if (source_p >= source_end_p)
    {
      parser_raise_error (context_p, PARSER_ERR_UNTERMINATED_REGEXP);
    }

    if (!in_class && source_p[0] == LIT_CHAR_SLASH)
    {
      regex_end_p = source_p;
      source_p++;
      column++;
      break;
    }

    switch (source_p[0])
    {
      case LIT_CHAR_CR:
      case LIT_CHAR_LF:
      case LEXER_NEWLINE_LS_PS_BYTE_1:
      {
        if (source_p[0] != LEXER_NEWLINE_LS_PS_BYTE_1 || LEXER_NEWLINE_LS_PS_BYTE_23 (source_p))
        {
          parser_raise_error (context_p, PARSER_ERR_NEWLINE_NOT_ALLOWED);
        }
        break;
      }
      case LIT_CHAR_TAB:
      {
        column = align_column_to_tab (column);
        /* Subtract -1 because column is increased below. */
        column--;
        break;
      }
      case LIT_CHAR_LEFT_SQUARE:
      {
        in_class = true;
        break;
      }
      case LIT_CHAR_RIGHT_SQUARE:
      {
        in_class = false;
        break;
      }
      case LIT_CHAR_BACKSLASH:
      {
        if (source_p + 1 >= source_end_p)
        {
          parser_raise_error (context_p, PARSER_ERR_UNTERMINATED_REGEXP);
        }

        if (source_p[1] >= 0x20 && source_p[1] <= LIT_UTF8_1_BYTE_CODE_POINT_MAX)
        {
          source_p++;
          column++;
        }
      }
    }

    source_p++;
    column++;

    while (source_p < source_end_p && IS_UTF8_INTERMEDIATE_OCTET (source_p[0]))
    {
      source_p++;
    }
  }

  current_flags = 0;
  while (source_p < source_end_p)
  {
    uint32_t flag = 0;

    if (source_p[0] == LIT_CHAR_LOWERCASE_G)
    {
      flag = RE_FLAG_GLOBAL;
    }
    else if (source_p[0] == LIT_CHAR_LOWERCASE_I)
    {
      flag = RE_FLAG_IGNORE_CASE;
    }
    else if (source_p[0] == LIT_CHAR_LOWERCASE_M)
    {
      flag = RE_FLAG_MULTILINE;
    }
    else if (source_p[0] == LIT_CHAR_LOWERCASE_U)
    {
      flag = RE_FLAG_UNICODE;
    }
    else if (source_p[0] == LIT_CHAR_LOWERCASE_Y)
    {
      flag = RE_FLAG_STICKY;
    }
#if JERRY_ESNEXT
    else if (source_p[0] == LIT_CHAR_LOWERCASE_S)
    {
      flag = RE_FLAG_DOTALL;
    }
#endif /* JERRY_ESNEXT */

    if (flag == 0)
    {
      break;
    }

    if (current_flags & flag)
    {
      parser_raise_error (context_p, PARSER_ERR_DUPLICATED_REGEXP_FLAG);
    }

    current_flags = (uint16_t) (current_flags | flag);
    source_p++;
    column++;
  }

  context_p->source_p = source_p;
  context_p->column = column;

  if (source_p < source_end_p && lexer_parse_identifier (context_p, LEXER_PARSE_CHECK_PART_AND_RETURN))
  {
    parser_raise_error (context_p, PARSER_ERR_UNKNOWN_REGEXP_FLAG);
  }

  length = (lit_utf8_size_t) (regex_end_p - regex_start_p);
  if (length > PARSER_MAXIMUM_STRING_LENGTH)
  {
    parser_raise_error (context_p, PARSER_ERR_REGEXP_TOO_LONG);
  }

  context_p->column = column;
  context_p->source_p = source_p;

  if (parse_only)
  {
    return;
  }

  if (context_p->literal_count >= PARSER_MAXIMUM_NUMBER_OF_LITERALS)
  {
    parser_raise_error (context_p, PARSER_ERR_LITERAL_LIMIT_REACHED);
  }

  /* Compile the RegExp literal and store the RegExp bytecode pointer */
  ecma_string_t *pattern_str_p = NULL;

  if (lit_is_valid_cesu8_string (regex_start_p, length))
  {
    pattern_str_p = ecma_new_ecma_string_from_utf8 (regex_start_p, length);
  }
  else
  {
    JERRY_ASSERT (lit_is_valid_utf8_string (regex_start_p, length, false));
    pattern_str_p = ecma_new_ecma_string_from_utf8_converted_to_cesu8 (regex_start_p, length);
  }

  re_compiled_code_t *re_bytecode_p = re_compile_bytecode (pattern_str_p, current_flags);
  ecma_deref_ecma_string (pattern_str_p);

  if (JERRY_UNLIKELY (re_bytecode_p == NULL))
  {
    parser_raise_error (context_p, PARSER_ERR_INVALID_REGEXP);
  }

  lexer_literal_t *literal_p = (lexer_literal_t *) parser_list_append (context_p, &context_p->literal_pool);
  literal_p->u.bytecode_p = (ecma_compiled_code_t *) re_bytecode_p;
  literal_p->type = LEXER_REGEXP_LITERAL;
  literal_p->prop.length = (prop_length_t) length;
  literal_p->status_flags = 0;

  context_p->token.type = LEXER_LITERAL;
  context_p->token.lit_location.type = LEXER_REGEXP_LITERAL;

  context_p->lit_object.literal_p = literal_p;
  context_p->lit_object.index = context_p->literal_count++;
#else /* !JERRY_BUILTIN_REGEXP */
  JERRY_UNUSED (parse_only);
  parser_raise_error (context_p, PARSER_ERR_UNSUPPORTED_REGEXP);
#endif /* JERRY_BUILTIN_REGEXP */
} /* lexer_construct_regexp_object */


lexer_convert_push_number_to_push_literal (parser_context_t *context_p) /**< context */
{
  ecma_integer_value_t value;
  bool two_literals = context_p->last_cbc_opcode >= CBC_PUSH_LITERAL_PUSH_NUMBER_0;

  if (context_p->last_cbc_opcode == CBC_PUSH_NUMBER_0 || context_p->last_cbc_opcode == CBC_PUSH_LITERAL_PUSH_NUMBER_0)
  {
    value = 0;
  }
  else if (context_p->last_cbc_opcode == CBC_PUSH_NUMBER_POS_BYTE
           || context_p->last_cbc_opcode == CBC_PUSH_LITERAL_PUSH_NUMBER_POS_BYTE)
  {
    value = ((ecma_integer_value_t) context_p->last_cbc.value) + 1;
  }
  else
  {
    JERRY_ASSERT (context_p->last_cbc_opcode == CBC_PUSH_NUMBER_NEG_BYTE
                  || context_p->last_cbc_opcode == CBC_PUSH_LITERAL_PUSH_NUMBER_NEG_BYTE);
    value = -((ecma_integer_value_t) context_p->last_cbc.value) - 1;
  }

  ecma_value_t lit_value = ecma_make_integer_value (value);

  parser_list_iterator_t literal_iterator;
  parser_list_iterator_init (&context_p->literal_pool, &literal_iterator);

  context_p->last_cbc_opcode = two_literals ? CBC_PUSH_TWO_LITERALS : CBC_PUSH_LITERAL;

  uint32_t literal_index = 0;
  lexer_literal_t *literal_p;

  while ((literal_p = (lexer_literal_t *) parser_list_iterator_next (&literal_iterator)) != NULL)
  {
    if (literal_p->type == LEXER_NUMBER_LITERAL && literal_p->u.value == lit_value)
    {
      if (two_literals)
      {
        context_p->last_cbc.value = (uint16_t) literal_index;
      }
      else
      {
        context_p->last_cbc.literal_index = (uint16_t) literal_index;
      }
      return;
    }

    literal_index++;
  }

  JERRY_ASSERT (literal_index == context_p->literal_count);

  if (literal_index >= PARSER_MAXIMUM_NUMBER_OF_LITERALS)
  {
    parser_raise_error (context_p, PARSER_ERR_LITERAL_LIMIT_REACHED);
  }

  literal_p = (lexer_literal_t *) parser_list_append (context_p, &context_p->literal_pool);
  literal_p->u.value = lit_value;
  literal_p->prop.length = 0; /* Unused. */
  literal_p->type = LEXER_NUMBER_LITERAL;
  literal_p->status_flags = 0;

  context_p->literal_count++;

  if (two_literals)
  {
    context_p->last_cbc.value = (uint16_t) literal_index;
  }
  else
  {
    context_p->last_cbc.literal_index = (uint16_t) literal_index;
  }
} /* lexer_convert_push_number_to_push_literal */


lexer_check_post_primary_exp (parser_context_t *context_p) /**< context */
{
  if (!(context_p->token.flags & LEXER_NO_SKIP_SPACES))
  {
    lexer_skip_spaces (context_p);
    context_p->token.flags = (uint8_t) (context_p->token.flags | LEXER_NO_SKIP_SPACES);
  }

  if (context_p->source_p >= context_p->source_end_p)
  {
    return false;
  }

  switch (context_p->source_p[0])
  {
    case LIT_CHAR_DOT:
    case LIT_CHAR_LEFT_PAREN:
    case LIT_CHAR_LEFT_SQUARE:
#if JERRY_ESNEXT
    case LIT_CHAR_GRAVE_ACCENT:
#endif /* JERRY_ESNEXT */
    {
      return true;
    }
    case LIT_CHAR_PLUS:
    case LIT_CHAR_MINUS:
    {
      return (!(context_p->token.flags & LEXER_WAS_NEWLINE) && context_p->source_p + 1 < context_p->source_end_p
              && context_p->source_p[1] == context_p->source_p[0]);
    }
#if JERRY_ESNEXT
    case LIT_CHAR_ASTERISK:
    {
      return (context_p->source_p + 1 < context_p->source_end_p
              && context_p->source_p[1] == (uint8_t) LIT_CHAR_ASTERISK);
    }
#endif /* JERRY_ESNEXT */
  }

  return false;
} /* lexer_check_post_primary_exp */


Don't constant-fold DT_RESOURCE constants.

PiperOrigin-RevId: 391803952
Change-Id: I0ea3ec31d3e7dfda0f03b4027a237f08d00a3091

bool MaybeReplaceShapeOp(
    const Node* n,
    const std::unordered_map<string, std::vector<PartialTensorShape>>*
        shape_map,
    std::unordered_map<const Node*, std::vector<Tensor>>*
        shape_replacement_map) {
  if (shape_map == nullptr || !IsShapeOp(n)) {
    return false;
  }
  // input_shapes will contain the shapes of each of n's inputs.
  std::vector<PartialTensorShape> input_shapes;
  if (!ReadPartialShapesFromShapeMap(n, shape_map, &input_shapes)) {
    return false;
  }
  const auto& ts = n->type_string();
  if (ts == "Shape" || ts == "ShapeN") {
    if (!MaybeReplaceShapeOrShapeNOp(n, input_shapes, shape_replacement_map)) {
      return false;
    }
  } else if (ts == "Rank") {
    if (!MaybeReplaceRankOp(n, input_shapes, shape_replacement_map)) {
      return false;
    }
  } else {
    CHECK_EQ(ts, "Size");
    if (!MaybeReplaceSizeOp(n, input_shapes, shape_replacement_map)) {
      return false;
    }
  }
  return true;
}

TensorFlow is an open source platform for machine learning. In affected versions during TensorFlow's Grappler optimizer phase, constant folding might attempt to deep copy a resource tensor. This results in a segfault, as these tensors are supposed to not change. The fix will be included in TensorFlow 2.7.0. We will also cherrypick this commit on TensorFlow 2.6.1, TensorFlow 2.5.2, and TensorFlow 2.4.4, as these are also affected and still in supported range.


bool ReadPartialShapesFromShapeMap(
    const Node* n,
    const std::unordered_map<string, std::vector<PartialTensorShape>>*
        shape_map,
    std::vector<PartialTensorShape>* input_shapes) {
  CHECK(shape_map != nullptr);
  input_shapes->resize(n->num_inputs());
  for (const Edge* in : n->in_edges()) {
    // Don't need to check if incoming control edges have known shapes.
    if (in->IsControlEdge()) continue;
    const auto known_shape_iter = shape_map->find(in->src()->name());
    if (known_shape_iter == shape_map->end()) {
      // One of n's inputs doesn't have known shapes, so don't replace n.
      return false;
    }
    const auto& known_shape = known_shape_iter->second;
    CHECK_GT(known_shape.size(), in->src_output()) << known_shape_iter->first;
    DCHECK_GE(in->dst_input(), 0);
    DCHECK_LT(in->dst_input(), input_shapes->size());
    (*input_shapes)[in->dst_input()] = known_shape[in->src_output()];
  }
  return true;
}


void AddShapeNodeToConstantGraph(
    Node* n,
    const std::unordered_map<const Node*, std::vector<Tensor>>&
        shape_replacement_map,
    std::unordered_map<Node*, std::vector<Node*>>* node_map,
    const ConstantFoldNameGenerator& generate_new_name, Graph* constant_graph) {
  std::vector<Node*>& added = (*node_map)[n];
  const string& node_name = n->name();
  for (const Tensor& t : shape_replacement_map.at(n)) {
    auto builder =
        NodeDefBuilder(generate_new_name(constant_graph, node_name), "Const")
            .Attr("dtype", t.dtype())
            .Attr("value", t);
    NodeDef def;
    CHECK(builder.Finalize(&def).ok());
    Node* constant_node;
    CHECK(NodeBuilder(builder).Finalize(constant_graph, &constant_node).ok());
    added.push_back(constant_node);
  }
  // Don't copy incoming edges to shape nodes that are being replaced.
}


bool ReplaceTensorWithConstant(
    Graph* graph, const Device* partition_device, NodeAndOutput tensor,
    const Tensor& constant, const gtl::FlatSet<Node*>& control_deps,
    int64_t max_constant_size_in_bytes,
    const ConstantFoldNameGenerator& generate_new_name) {
  // Be conservative when replacing a tensor with a constant, when not
  // running on CPU.
  // 1) Do not replace another constant.
  // 2) If the destination tensor or any other tensor from the same node is not
  // an int32 tensor, and has HOST_MEMORY constraint, do not replace it.
  // 3) If the destination tensor or any other tensor from the same node is an
  // int32 tensor, and has DEVICE_MEMORY constraint, do not replace it.
  // 4) If the size of the constant in bytes is too large (>
  // max_constant_in_bytes), do not replace it. This prevents the size of the
  // Graph from growing too large.
  // 5) If the constant op created does not have a kernel implementation
  // for the device, do not use it.
  // TODO(keveman): Consider adding a new constant op that has a kernel
  // implementation for all types, but with HostMemory constraint on it's
  // output.
  if (tensor.first->IsConstant()) {
    return false;
  }
  DeviceType device_type = partition_device
                               ? DeviceType{partition_device->device_type()}
                               : DEVICE_CPU;
  if (partition_device && device_type != DEVICE_CPU) {
    MemoryTypeVector input_mvec;
    MemoryTypeVector output_mvec;
    if (!MemoryTypesForNode(graph->op_registry(), device_type,
                            tensor.first->def(), &input_mvec, &output_mvec)
             .ok()) {
      return false;
    }
    for (int i = 0; i < output_mvec.size(); i++) {
      MemoryType memory_type = output_mvec[i];
      bool is_int32 = tensor.first->output_type(i) == DT_INT32;
      if ((memory_type == HOST_MEMORY && !is_int32) ||
          (memory_type == DEVICE_MEMORY && is_int32)) {
        return false;
      }
    }
  }
  if (constant.TotalBytes() > max_constant_size_in_bytes) {
    return false;
  }

  Node* n = tensor.first;
  std::vector<const Edge*> edges_to_remove;
  for (const Edge* out_edge : n->out_edges()) {
    if (out_edge->src_output() == tensor.second) {
      edges_to_remove.push_back(out_edge);
    }
  }
  const string& node_name = n->name();
  Node* constant_node;
  auto builder = NodeDefBuilder(generate_new_name(graph, node_name), "Const")
                     .Attr("dtype", constant.dtype())
                     .Attr("value", constant);
  if (partition_device) {
    builder.Device(partition_device->name());
  }
  NodeDef def;
  if (!builder.Finalize(&def).ok()) {
    return false;
  }
  const KernelDef* kdef;
  if (!FindKernelDef(device_type, def, &kdef, nullptr).ok()) {
    return false;
  }

  VLOG(1) << "Replacing " << tensor.first->name() << " :: " << tensor.second
          << " with a constant";

  if (!NodeBuilder(builder).Finalize(graph, &constant_node).ok()) {
    return false;
  }
  for (auto edge : edges_to_remove) {
    graph->AddEdge(constant_node, 0, edge->dst(), edge->dst_input());
    graph->RemoveEdge(edge);
  }
  if (control_deps.empty()) {
    graph->AddControlEdge(graph->source_node(), constant_node);
  } else {
    for (Node* node : control_deps) {
      graph->AddControlEdge(node, constant_node);
    }
  }
  if (partition_device) {
    constant_node->set_assigned_device_name(partition_device->name());
  }
  return true;
}


Graph* GetConstantGraph(
    const Graph* orig_graph, const std::vector<Node*>& nodes,
    const std::unordered_map<const Node*, std::vector<Tensor>>&
        shape_replacement_map,
    std::map<NodeAndOutput, NodeAndOutput>* tensors_to_fetch,
    const ConstantFoldNameGenerator& generate_new_name) {
  Graph* constant_graph = new Graph(orig_graph->op_registry());
  std::unordered_map<Node*, std::vector<Node*>> node_map;
  node_map[orig_graph->source_node()] = {constant_graph->source_node()};
  node_map[orig_graph->sink_node()] = {constant_graph->sink_node()};
  for (Node* n : nodes) {
    if (shape_replacement_map.count(n) == 0) {
      AddNodeToConstantGraph(n, &node_map, constant_graph);
    } else {
      AddShapeNodeToConstantGraph(n, shape_replacement_map, &node_map,
                                  generate_new_name, constant_graph);
    }
  }

  for (auto const& added_nodes : node_map) {
    for (const Edge* out_edge : added_nodes.first->out_edges()) {
      if (node_map.count(out_edge->dst()) == 0) {
        if (out_edge->IsControlEdge()) continue;
        if (added_nodes.second.size() == 1) {
          tensors_to_fetch->insert(
              {{added_nodes.second[0], out_edge->src_output()},
               {added_nodes.first, out_edge->src_output()}});
        } else {
          // The node had multiple outputs and was replaced by a
          // vector of constants, so the NodeAndOutput is the 0th
          // output of the kth added constant, rather than the kth
          // output of the added node as in the standard case above.
          tensors_to_fetch->insert(
              {{added_nodes.second[out_edge->src_output()], 0},
               {added_nodes.first, out_edge->src_output()}});
        }
      }
    }
  }

  return constant_graph;
}


void ConsiderConstantFoldableNode(
    Node* n, const ConstantFoldingOptions& opts, std::vector<Node*>* nodes,
    std::unordered_map<const Node*, gtl::FlatSet<Node*>>* constant_control_deps,
    std::unordered_map<const Node*, std::vector<Tensor>>* shape_replacement_map,
    bool* internal_node_inserted) {
  if (IsConstantFoldable(n, opts.shape_map, opts.consider,
                         opts.max_constant_size_in_bytes,
                         shape_replacement_map)) {
    // A node is constant provided all of its non-control incoming Tensors come
    // from constant nodes, or it's a shape Op with statically known inputs in
    // which case it is placed in shape_replacement_map.
    //
    // We allow control dependencies from non-constant nodes to constant nodes,
    // but to preserve the graph structure we must transfer the control
    // dependency onto any constant replacement.
    bool all_parents_constant = true;
    for (const Edge* in : n->in_edges()) {
      // Allows non-constant -> constant control edges.
      if (!in->IsControlEdge() &&
          constant_control_deps->count(in->src()) == 0) {
        all_parents_constant = false;
        break;
      }
    }
    if (all_parents_constant || shape_replacement_map->count(n) != 0) {
      gtl::FlatSet<Node*>& control_deps = (*constant_control_deps)[n];
      for (const Edge* e : n->in_edges()) {
        if (constant_control_deps->count(e->src()) == 0) {
          // This branch is taken if the incoming edge is a control dependency,
          // in which case we want to add it to the dependencies being
          // accumulated for this node, or the incoming edge is not
          // constant. The latter may happen when n is a shape node and the
          // source has known shape. In that case add a control dependency from
          // the source node, since there was previously a data dependency and
          // we want to preserve sequencing constraints.
          if (!e->src()->IsSource()) {
            control_deps.insert(e->src());
          }
        } else {
          // If the parent has been accumulating control dependencies, add all
          // of its transitive control deps.
          const gtl::FlatSet<Node*>& parent_deps =
              (*constant_control_deps)[e->src()];
          control_deps.insert(parent_deps.begin(), parent_deps.end());
        }
      }
      nodes->push_back(n);
      if (!n->IsConstant()) {
        *internal_node_inserted = true;
      }
    }
  }
}


bool IsShapeOp(const Node* n) {
  const auto& ts = n->type_string();
  return ts == "Shape" || ts == "ShapeN" || ts == "Rank" || ts == "Size";
}


void AddNodeToConstantGraph(
    Node* n, std::unordered_map<Node*, std::vector<Node*>>* node_map,
    Graph* constant_graph) {
  std::vector<Node*>& added = (*node_map)[n];
  added.push_back(constant_graph->CopyNode(n));
  for (const Edge* in_edge : n->in_edges()) {
    // Don't copy control edges to the constant graph.
    if (!in_edge->IsControlEdge()) {
      Node* in = in_edge->src();
      auto it = node_map->find(in);
      CHECK(it != node_map->end())
          << n->DebugString() << " <-" << in->DebugString();
      if (it->second.size() == 1) {
        constant_graph->AddEdge(it->second[0], in_edge->src_output(), added[0],
                                in_edge->dst_input());
      } else {
        // The original source node had multiple outputs and was replaced by a
        // vector of constants, so the edge comes from the 0th output of the kth
        // added constant, rather than the kth output of the added node as in
        // the standard case above.
        constant_graph->AddEdge(it->second[in_edge->src_output()], 0, added[0],
                                in_edge->dst_input());
      }
    }
  }
}


bool MaybeReplaceShapeOrShapeNOp(
    const Node* n, const std::vector<PartialTensorShape>& input_shapes,
    std::unordered_map<const Node*, std::vector<Tensor>>*
        shape_replacement_map) {
  std::vector<Tensor> defined_shape;
  for (const auto& shape : input_shapes) {
    if (!shape.IsFullyDefined()) {
      return false;
    }
    const int rank = shape.dims();
    DataType op_type = n->output_type(0);
    Tensor t(op_type, TensorShape({rank}));
    if (op_type == DT_INT64) {
      auto vec = t.vec<int64_t>();
      for (int i = 0; i < rank; ++i) {
        vec(i) = shape.dim_size(i);
      }
    } else {
      CHECK(op_type == DT_INT32);
      auto vec = t.vec<int32>();
      for (int i = 0; i < rank; ++i) {
        if (shape.dim_size(i) > INT_MAX) {
          VLOG(1) << "Node " << n->name() << " has input shape dimension " << i
                  << " of " << shape.dim_size(i) << " but type INT32 "
                  << " so not replacing as constant: this will trigger a "
                     "runtime error later.";
          return false;
        }
        vec(i) = static_cast<int32>(shape.dim_size(i));
      }
    }
    defined_shape.push_back(t);
  }
  // All the inputs had known shapes so we can replace the node by constants
  // later in the rewrite.
  shape_replacement_map->insert({n, defined_shape});
  return true;
}


int64_t UniqueConstantId() {
  static std::atomic_int_fast64_t unique_constant_id;
  return unique_constant_id.fetch_add(1);
}


bool IsConstantFoldable(
    const Node* n,
    const std::unordered_map<string, std::vector<PartialTensorShape>>*
        shape_map,
    const std::function<bool(const Node*)>& consider,
    int64_t max_constant_size_in_bytes,
    std::unordered_map<const Node*, std::vector<Tensor>>*
        shape_replacement_map) {
  if (n->IsConstant()) {
    // Skip constant folding resources as they cannot be deep copied.
    return n->output_type(0) != DT_RESOURCE;
  }
  if (MaybeReplaceShapeOp(n, shape_map, shape_replacement_map)) {
    return true;
  }
  if (n->op_def().is_stateful()) {
    return false;
  }
  if (consider && !consider(n)) {
    return false;
  }
  if (shape_map != nullptr) {
    // We can skip the node if an output is known to be oversized.
    auto shape_it = shape_map->find(n->name());
    if (shape_it != shape_map->end()) {
      for (int64_t i = 0; i < shape_it->second.size(); ++i) {
        const auto& out_shape = shape_it->second[i];
        if (out_shape.IsFullyDefined() &&
            out_shape.num_elements() * DataTypeSize(n->output_type(i)) >
                max_constant_size_in_bytes) {
          return false;
        }
      }
    }
  }
  if (n->IsControlFlow() || n->IsSend() || n->IsRecv()) {
    return false;
  }
  // TODO(yuanbyu): For now disable these session handle operations.
  if (n->IsGetSessionHandle() || n->IsGetSessionTensor() ||
      n->IsDeleteSessionTensor()) {
    return false;
  }
  if (n->IsSource()) {
    return false;
  }
  if (n->IsSink()) {
    return false;
  }
  if (n->IsFakeParam()) {
    return false;
  }
  // Since constant-folding runs on the CPU, do not attempt to constant-fold
  // operators that have no CPU kernel. Also implies that we will not
  // constant-fold functions.
  // TODO(phawkins): allow constant-folding for functions; functions may
  // be arbitrarily expensive to execute.
  if (!KernelDefAvailable(DeviceType(DEVICE_CPU), n->def())) {
    return false;
  }
  // Do not constant fold nodes which will be allocated by ScopedAllocator.
  // This is because the constant-folding graph will not contain the
  // `_ScopedAllocator` node, and that is necessary to be able to run a node
  // that will use this allocator.
  if (n->attrs().Find(kScopedAllocatorAttrName) != nullptr) {
    VLOG(2) << "Skip node [" << n->DebugString()
            << "] for constant folding due to scoped allocator";
    return false;
  }
  return true;
}


Status ConstantFold(const ConstantFoldingOptions& opts,
                    FunctionLibraryRuntime* function_library, Env* env,
                    const Device* partition_device, Graph* graph,
                    bool* was_mutated) {
  // TensorFlow flushes denormals to zero and rounds to nearest, so we do
  // the same here.
  port::ScopedFlushDenormal flush;
  port::ScopedSetRound round(FE_TONEAREST);

  DumpGraph("Before", graph);
  ConstantFoldNameGenerator generate_new_name = opts.generate_new_name;
  if (generate_new_name == nullptr) {
    generate_new_name = [](Graph* graph, string old_name) {
      return strings::StrCat(graph->NewName(old_name), "__cf__",
                             UniqueConstantId());
    };
  }

  std::vector<Node*> constant_foldable_nodes;
  std::unordered_map<const Node*, gtl::FlatSet<Node*>> constant_control_deps;
  std::unordered_map<const Node*, std::vector<Tensor>> shape_replacement_map;
  FindConstantFoldableNodes(graph, opts, &constant_foldable_nodes,
                            &constant_control_deps, &shape_replacement_map);
  if (constant_foldable_nodes.empty()) {
    VLOG(1) << "No constant foldable nodes found";
    *was_mutated = false;
    // This is not an error, so return the status as OK.
    return Status::OK();
  }

  std::map<NodeAndOutput, NodeAndOutput> tensors_to_fetch;
  std::unique_ptr<Graph> constant_graph(
      GetConstantGraph(graph, constant_foldable_nodes, shape_replacement_map,
                       &tensors_to_fetch, generate_new_name));
  DumpGraph("Constant graph", constant_graph.get());

  if (tensors_to_fetch.empty()) {
    VLOG(1) << "No constant nodes found that feed into the original graph.";
    *was_mutated = false;
    // This is not an error, so return the status as OK.
    return Status::OK();
  }
  VLOG(1) << "Constant foldable " << constant_graph->num_node_ids() << " : "
          << graph->num_node_ids();

  std::vector<string> tensors_to_fetch_names;
  std::vector<NodeAndOutput> tensors_to_replace;
  // Sorting the nodes based on the name gives us a stable ordering between runs
  // for the same graph.
  std::vector<std::pair<NodeAndOutput, NodeAndOutput>> tensors_to_fetch_sorted(
      tensors_to_fetch.begin(), tensors_to_fetch.end());
  std::sort(tensors_to_fetch_sorted.begin(), tensors_to_fetch_sorted.end(),
            [](const std::pair<NodeAndOutput, NodeAndOutput>& n1,
               const std::pair<NodeAndOutput, NodeAndOutput>& n2) {
              return std::tie(n1.first.first->name(), n1.first.second) <
                     std::tie(n2.first.first->name(), n2.first.second);
            });
  for (auto n : tensors_to_fetch_sorted) {
    tensors_to_fetch_names.push_back(
        strings::StrCat(n.first.first->name(), ":", n.first.second));
    tensors_to_replace.push_back(n.second);
  }

  auto graph_runner = std::unique_ptr<GraphRunner>(new GraphRunner(env));
  // Evaluate the constant foldable nodes.
  std::vector<Tensor> outputs;
  auto delete_tensors = gtl::MakeCleanup([&graph_runner, &outputs] {
    // Output tensors need to be cleared before the GraphRunner is deleted.
    outputs.clear();
    graph_runner.reset(nullptr);
  });

  Status s =
      graph_runner->Run(constant_graph.get(), function_library, {} /* inputs*/,
                        tensors_to_fetch_names, &outputs);
  if (!s.ok()) {
    VLOG(1) << "Could not fetch constants: " << s;
    *was_mutated = false;
    return s;
  }

  // Fetch the constant tensors and replace the corresponding tensors in the
  // original graph with those constants.
  int32_t num_nodes_replaced = 0;
  for (size_t c = 0; c < outputs.size(); ++c) {
    const gtl::FlatSet<Node*>& control_deps =
        constant_control_deps[tensors_to_replace[c].first];
    if (ReplaceTensorWithConstant(
            graph, partition_device, tensors_to_replace[c], outputs[c],
            control_deps, opts.max_constant_size_in_bytes, generate_new_name)) {
      ++num_nodes_replaced;
    }
  }

  DumpGraph("After", graph);

  *was_mutated = (num_nodes_replaced > 0);
  return Status::OK();
}


bool MaybeReplaceSizeOp(const Node* n,
                        const std::vector<PartialTensorShape>& input_shapes,
                        std::unordered_map<const Node*, std::vector<Tensor>>*
                            shape_replacement_map) {
  CHECK_EQ(input_shapes.size(), 1);
  if (!input_shapes[0].IsFullyDefined()) {
    return false;
  }
  DataType op_type = n->output_type(0);
  Tensor t(op_type, TensorShape({}));
  int64_t size = input_shapes[0].num_elements();
  if (op_type == DT_INT64) {
    t.scalar<int64_t>()() = size;
  } else {
    CHECK(op_type == DT_INT32);
    if (size > INT_MAX) {
      VLOG(1) << "Node " << n->name() << " has input shape size " << size
              << " but type INT32 "
              << " so not replacing as constant: this will trigger a runtime "
                 "error later.";
      return false;
    }
    t.scalar<int32>()() = static_cast<int32>(size);
  }
  shape_replacement_map->insert({n, {t}});
  return true;
}


void FindConstantFoldableNodes(
    const Graph* graph, const ConstantFoldingOptions& opts,
    std::vector<Node*>* nodes,
    std::unordered_map<const Node*, gtl::FlatSet<Node*>>* constant_control_deps,
    std::unordered_map<const Node*, std::vector<Tensor>>*
        shape_replacement_map) {
  bool internal_node_inserted = false;
  // Walk the nodes in data flow order.
  ReverseDFS(
      *graph, nullptr,
      [nodes, constant_control_deps, shape_replacement_map,
       &internal_node_inserted, &opts](Node* n) {
        ConsiderConstantFoldableNode(n, opts, nodes, constant_control_deps,
                                     shape_replacement_map,
                                     &internal_node_inserted);
      },
      NodeComparatorName());
  // If we have inserted just leaf level nodes, then there is nothing to fold.
  if (!internal_node_inserted) {
    nodes->clear();
    constant_control_deps->clear();
  }
}


bool MaybeReplaceRankOp(const Node* n,
                        const std::vector<PartialTensorShape>& input_shapes,
                        std::unordered_map<const Node*, std::vector<Tensor>>*
                            shape_replacement_map) {
  CHECK_EQ(input_shapes.size(), 1);
  if (input_shapes[0].unknown_rank()) {
    return false;
  }
  Tensor t(DT_INT32, TensorShape({}));
  t.scalar<int32>()() = input_shapes[0].dims();
  shape_replacement_map->insert({n, {t}});
  return true;
}


Properly fix the UAF in r_io_bank_map_add_top ##crash

* Associated with the CVE-2022-0559
* Reported by alkyne Choi via huntr.dev

R_API bool r_io_bank_map_add_top(RIO *io, const ut32 bankid, const ut32 mapid) {
	RIOBank *bank = r_io_bank_get (io, bankid);
	RIOMap *map = r_io_map_get (io, mapid);
	r_return_val_if_fail (io && bank && map, false);
	RIOMapRef *mapref = _mapref_from_map (map);
	if (!mapref) {
		return false;
	}
	RIOSubMap *sm = r_io_submap_new (io, mapref);
	if (!sm) {
		free (mapref);
		return false;
	}
	RRBNode *entry = _find_entry_submap_node (bank, sm);
	if (!entry) {
		// no intersection with any submap, so just insert
		if (!r_crbtree_insert (bank->submaps, sm, _find_sm_by_from_vaddr_cb, NULL)) {
			free (sm);
			free (mapref);
			return false;
		}
		r_list_append (bank->maprefs, mapref);
		return true;
	}
	bank->last_used = NULL;
	RIOSubMap *bd = (RIOSubMap *)entry->data;
	if (r_io_submap_to (bd) == r_io_submap_to (sm) &&
		r_io_submap_from (bd) >= r_io_submap_from (sm)) {
		// _find_entry_submap_node guarantees, that there is no submap
		// prior to bd in the range of sm, so instead of deleting and inserting
		// we can just memcpy
		memcpy (bd, sm, sizeof (RIOSubMap));
		free (sm);
		r_list_append (bank->maprefs, mapref);
		return true;
	}
	if (r_io_submap_from (bd) < r_io_submap_from (sm) &&
		r_io_submap_to (sm) < r_io_submap_to (bd)) {
		// split bd into 2 maps => bd and bdsm
		RIOSubMap *bdsm = R_NEWCOPY (RIOSubMap, bd);
		if (!bdsm) {
			free (sm);
			free (mapref);
			return false;
		}
		r_io_submap_set_from (bdsm, r_io_submap_to (sm) + 1);
		r_io_submap_set_to (bd, r_io_submap_from (sm) - 1);
		// TODO: insert and check return value, before adjusting sm size
		if (!r_crbtree_insert (bank->submaps, sm, _find_sm_by_from_vaddr_cb, NULL)) {
			free (sm);
			free (bdsm);
			free (mapref);
			return false;
		}
		if (!r_crbtree_insert (bank->submaps, bdsm, _find_sm_by_from_vaddr_cb, NULL)) {
			r_crbtree_delete (bank->submaps, sm, _find_sm_by_from_vaddr_cb, NULL);
			free (sm);
			free (bdsm);
			free (mapref);
			return false;
		}
		r_list_append (bank->maprefs, mapref);
		return true;
	}

	// guaranteed intersection
	if (r_io_submap_from (bd) < r_io_submap_from (sm)) {
		r_io_submap_set_to (bd, r_io_submap_from (sm) - 1);
		entry = r_rbnode_next (entry);
	}
	ut64 smto = r_io_submap_to (sm);
	if (entry) {
		ut64 ento = r_io_submap_to (((RIOSubMap*)entry->data));
		while (entry && ento <= smto) {
			//delete all submaps that are completly included in sm
			RRBNode *next = r_rbnode_next (entry);
			// this can be optimized, there is no need to do search here
			// XXX this is a workaround to avoid an UAF in Reproducer: iobank-crash
			ento = r_io_submap_to (((RIOSubMap*)entry->data));
			bool a = r_crbtree_delete (bank->submaps, entry->data, _find_sm_by_from_vaddr_cb, NULL);
			if (!a) {
				next = NULL;
			}
			entry = next;
		}
	}
	if (entry && r_io_submap_from (((RIOSubMap *)entry->data)) <= r_io_submap_to (sm)) {
		bd = (RIOSubMap *)entry->data;
		r_io_submap_set_from (bd, r_io_submap_to (sm) + 1);
	}
	if (!r_crbtree_insert (bank->submaps, sm, _find_sm_by_from_vaddr_cb, NULL)) {
		free (sm);
		free (mapref);
		return false;
	}
	r_list_append (bank->maprefs, mapref);
	return true;
}


security/41: Don't perform PROMPT_SUBST evaluation on %F/%K arguments

Mitigates CVE-2021-45444

stradd(char *d)
{
#ifdef MULTIBYTE_SUPPORT
    char *ums, *ups;
    int upslen, eol = 0;
    mbstate_t mbs;

    memset(&mbs, 0, sizeof mbs);
    ums = ztrdup(d);
    ups = unmetafy(ums, &upslen);

    /*
     * We now have a raw string of possibly multibyte characters.
     * Read each character one by one.
     */
    while (upslen > 0) {
	wchar_t cc;
	char *pc;
	size_t cnt = eol ? MB_INVALID : mbrtowc(&cc, ups, upslen, &mbs);

	switch (cnt) {
	case MB_INCOMPLETE:
	    eol = 1;
	    /* FALL THROUGH */
	case MB_INVALID:
	    /* Bad character.  Take the next byte on its own. */
	    pc = nicechar(*ups);
	    cnt = 1;
	    memset(&mbs, 0, sizeof mbs);
	    break;
	case 0:
	    cnt = 1;
	    /* FALL THROUGH */
	default:
	    /* Take full wide character in one go */
	    mb_charinit();
	    pc = wcs_nicechar(cc, NULL, NULL);
	    break;
	}
	/* Keep output as metafied string. */
	addbufspc(strlen(pc));

	upslen -= cnt;
	ups += cnt;

	/* Put printed representation into the buffer */
	while (*pc)
	    *bv->bp++ = *pc++;
    }

    free(ums);
#else
    char *ps, *pc;
    addbufspc(niceztrlen(d));
    /* This loop puts the nice representation of the string into the
     * prompt buffer. */
    for (ps = d; *ps; ps++) {
	for (pc = nicechar(*ps == Meta ? *++ps^32 : *ps); *pc; pc++)
	    *bv->bp++ = *pc;
    }
#endif
}


putstr(int d)
{
    addbufspc(1);
    pputc(d);
    return 0;
}


allocate_colour_buffer(void)
{
    char **atrs;
    int lenfg, lenbg, len;

    if (colseq_buf_allocs++)
	return;

    atrs = getaparam("zle_highlight");
    if (atrs) {
	for (; *atrs; atrs++) {
	    if (strpfx("fg_start_code:", *atrs)) {
		set_colour_code(*atrs + 14, &fg_bg_sequences[COL_SEQ_FG].start);
	    } else if (strpfx("fg_default_code:", *atrs)) {
		set_colour_code(*atrs + 16, &fg_bg_sequences[COL_SEQ_FG].def);
	    } else if (strpfx("fg_end_code:", *atrs)) {
		set_colour_code(*atrs + 12, &fg_bg_sequences[COL_SEQ_FG].end);
	    } else if (strpfx("bg_start_code:", *atrs)) {
		set_colour_code(*atrs + 14, &fg_bg_sequences[COL_SEQ_BG].start);
	    } else if (strpfx("bg_default_code:", *atrs)) {
		set_colour_code(*atrs + 16, &fg_bg_sequences[COL_SEQ_BG].def);
	    } else if (strpfx("bg_end_code:", *atrs)) {
		set_colour_code(*atrs + 12, &fg_bg_sequences[COL_SEQ_BG].end);
	    }
	}
    }

    lenfg = strlen(fg_bg_sequences[COL_SEQ_FG].def);
    /* always need 1 character for non-default code */
    if (lenfg < 1)
	lenfg = 1;
    lenfg += strlen(fg_bg_sequences[COL_SEQ_FG].start) +
	strlen(fg_bg_sequences[COL_SEQ_FG].end);

    lenbg = strlen(fg_bg_sequences[COL_SEQ_BG].def);
    /* always need 1 character for non-default code */
    if (lenbg < 1)
	lenbg = 1;
    lenbg += strlen(fg_bg_sequences[COL_SEQ_BG].start) +
	strlen(fg_bg_sequences[COL_SEQ_BG].end);

    len = lenfg > lenbg ? lenfg : lenbg;
    /* add 1 for the null and 14 for truecolor */
    colseq_buf = (char *)zalloc(len+15);
}


free_colour_buffer(void)
{
    if (--colseq_buf_allocs)
	return;

    DPUTS(!colseq_buf, "Freeing colour sequence buffer without alloc");
    /* Free buffer for colour code composition */
    free(colseq_buf);
    colseq_buf = NULL;
}


match_highlight(const char *teststr, zattr *on_var)
{
    int found = 1;

    *on_var = 0;
    while (found && *teststr) {
	const struct highlight *hl;

	found = 0;
	if (strpfx("fg=", teststr) || strpfx("bg=", teststr)) {
	    int is_fg = (teststr[0] == 'f');
	    zattr atr;

	    teststr += 3;
	    atr = match_colour(&teststr, is_fg, 0);
	    if (*teststr == ',')
		teststr++;
	    else if (*teststr)
		break;
	    found = 1;
	    /* skip out of range colours but keep scanning attributes */
	    if (atr != TXT_ERROR)
		*on_var |= atr;
	} else {
	    for (hl = highlights; hl->name; hl++) {
		if (strpfx(hl->name, teststr)) {
		    const char *val = teststr + strlen(hl->name);

		    if (*val == ',')
			val++;
		    else if (*val)
			break;

		    *on_var |= hl->mask_on;
		    *on_var &= ~hl->mask_off;
		    teststr = val;
		    found = 1;
		}
	    }
	}
    }
}


pputc(char c)
{
    if (imeta(c)) {
	*bv->bp++ = Meta;
	c ^= 32;
    }
    *bv->bp++ = c;
    if (c == '\n' && !bv->dontcount)
	bv->bufline = bv->bp;
}


set_colour_code(char *str, char **var)
{
    char *keyseq;
    int len;

    zsfree(*var);
    keyseq = getkeystring(str, &len, GETKEYS_BINDKEY, NULL);
    *var = metafy(keyseq, len, META_DUP);
}


set_default_colour_sequences(void)
{
    fg_bg_sequences[COL_SEQ_FG].start = ztrdup(TC_COL_FG_START);
    fg_bg_sequences[COL_SEQ_FG].end = ztrdup(TC_COL_FG_END);
    fg_bg_sequences[COL_SEQ_FG].def = ztrdup(TC_COL_FG_DEFAULT);

    fg_bg_sequences[COL_SEQ_BG].start = ztrdup(TC_COL_BG_START);
    fg_bg_sequences[COL_SEQ_BG].end = ztrdup(TC_COL_BG_END);
    fg_bg_sequences[COL_SEQ_BG].def = ztrdup(TC_COL_BG_DEFAULT);
}


match_named_colour(const char **teststrp)
{
    const char *teststr = *teststrp, *end, **cptr;
    int len;

    for (end = teststr; ialpha(*end); end++)
	;
    len = end - teststr;
    *teststrp = end;

    for (cptr = ansi_colours; *cptr; cptr++) {
	if (!strncmp(teststr, *cptr, len))
	    return cptr - ansi_colours;
    }

    return -1;
}


output_highlight(zattr atr, char *buf)
{
    const struct highlight *hp;
    int atrlen = 0, len;
    char *ptr = buf;

    if (atr & TXTFGCOLOUR) {
	len = output_colour(txtchangeget(atr, TXT_ATTR_FG_COL),
			    COL_SEQ_FG,
			    (atr & TXT_ATTR_FG_TERMCAP),
			    (atr & TXT_ATTR_FG_24BIT),
			    ptr);
	atrlen += len;
	if (buf)
	    ptr += len;
    }
    if (atr & TXTBGCOLOUR) {
	if (atrlen) {
	    atrlen++;
	    if (buf) {
		strcpy(ptr, ",");
		ptr++;
	    }
	}
	len = output_colour(txtchangeget(atr, TXT_ATTR_BG_COL),
			    COL_SEQ_BG,
			    (atr & TXT_ATTR_BG_TERMCAP),
			    (atr & TXT_ATTR_BG_24BIT),
			    ptr);
	atrlen += len;
	if (buf)
	    ptr += len;
    }
    for (hp = highlights; hp->name; hp++) {
	if (hp->mask_on & atr) {
	    if (atrlen) {
		atrlen++;
		if (buf) {
		    strcpy(ptr, ",");
		    ptr++;
		}
	    }
	    len = strlen(hp->name);
	    atrlen += len;
	    if (buf) {
		strcpy(ptr, hp->name);
		ptr += len;
	    }
	}
    }

    if (atrlen == 0) {
	if (buf)
	    strcpy(ptr, "none");
	return 4;
    }
    return atrlen;
}


output_colour(int colour, int fg_bg, int use_tc, int truecol, char *buf)
{
    int atrlen = 3, len;
    char *ptr = buf;
    if (buf) {
	strcpy(ptr, fg_bg == COL_SEQ_FG ? "fg=" : "bg=");
	ptr += 3;
    }
    if (truecol) {
	/* length of hex triplet always 7, don't need sprintf to count */
	atrlen += buf ? sprintf(ptr, "#%02x%02x%02x", colour >> 16,
		(colour >> 8) & 0xff, colour & 0xff) : 7;
    /* colour should only be > 7 if using termcap but let's be safe. Update:
     * currently other places in code don't always imply that colour > 7 =>
     * using-termcap - if zle_highlight will be non-default, then it will be
     * used instead of termcap even for colour > 7. Here this just emits the
     * color number, so it works fine for both zle_highlight and tercap cases
     */
    } else if (use_tc || colour > 7) {
	char digbuf[DIGBUFSIZE];
	sprintf(digbuf, "%d", colour);
	len = strlen(digbuf);
	atrlen += len;
	if (buf)
	    strcpy(ptr, digbuf);
    } else {
	len = strlen(ansi_colours[colour]);
	atrlen += len;
	if (buf)
	    strcpy(ptr, ansi_colours[colour]);
    }

    return atrlen;
}


countprompt(char *str, int *wp, int *hp, int overf)
{
    int w = 0, h = 1, multi = 0, wcw = 0;
    int s = 1;
#ifdef MULTIBYTE_SUPPORT
    char inchar;
    mbstate_t mbs;
    wchar_t wc;

    memset(&mbs, 0, sizeof(mbs));
#endif

    for (; *str; str++) {
	/*
	 * Avoid double-incrementing the height when there's a newline in the
	 * prompt and the line it terminates takes up exactly the width of the
	 * terminal
	 */
	while (w > zterm_columns && overf >= 0 && !multi) {
	    h++;
	    if (wcw) {
		/*
		 * Wide characters don't get split off. They move to the
		 * next line if there is not enough space.
		 */
		w = wcw;
		break;
	    } else {
		/*
		 * Tabs overflow to the next line as if they were made of spaces.
		 */
		w -= zterm_columns;
	    }
	}
	wcw = 0;
	/*
	 * Input string should be metafied, so tokens in it should
	 * be real tokens, even if there are multibyte characters.
	 */
	if (*str == Inpar)
	    s = 0;
	else if (*str == Outpar)
	    s = 1;
	else if (*str == Nularg)
	    w++;
	else if (s) {
	    if (*str == Meta) {
#ifdef MULTIBYTE_SUPPORT
		inchar = *++str ^ 32;
#else
		str++;
#endif
	    } else {
#ifdef MULTIBYTE_SUPPORT
		/*
		 * Don't look for tab or newline in the middle
		 * of a multibyte character.  Otherwise, we are
		 * relying on the character set being an extension
		 * of ASCII so it's safe to test a single byte.
		 */
		if (!multi) {
#endif
		    if (*str == '\t') {
			w = (w | 7) + 1;
			continue;
		    } else if (*str == '\n') {
			w = 0;
			h++;
			continue;
		    }
#ifdef MULTIBYTE_SUPPORT
		}

		inchar = *str;
#endif
	    }

#ifdef MULTIBYTE_SUPPORT
	    switch (mbrtowc(&wc, &inchar, 1, &mbs)) {
	    case MB_INCOMPLETE:
		/* Character is incomplete -- keep looking. */
		multi = 1;
		break;
	    case MB_INVALID:
		memset(&mbs, 0, sizeof mbs);
		/* Invalid character: assume single width. */
		multi = 0;
		w++;
		break;
	    case 0:
		multi = 0;
		break;
	    default:
		/*
		 * If the character isn't printable, WCWIDTH() returns
		 * -1.  We assume width 1.
		 */
		wcw = WCWIDTH(wc);
		if (wcw >= 0)
		    w += wcw;
		else
		    w++;
		multi = 0;
		break;
	    }
#else
	    w++;
#endif
	}
    }
    /*
     * multi may still be set if we were in the middle of the character.
     * This isn't easy to handle generally; just assume there's no
     * output.
     */
    while (w > zterm_columns && overf >= 0) {
	h++;
	if (wcw) {
	    w = wcw;
	    break;
	} else {
	    w -= zterm_columns;
	}
    }
    if (w == zterm_columns && overf == 0) {
	w = 0;
	h++;
    }
    if(wp)
	*wp = w;
    if(hp)
	*hp = h;
}


promptpath(char *p, int npath, int tilde)
{
    char *modp = p;
    Nameddir nd;

    if (tilde && ((nd = finddir(p))))
	modp = tricat("~", nd->node.nam, p + strlen(nd->dir));

    if (npath) {
	char *sptr;
	if (npath > 0) {
	    for (sptr = modp + strlen(modp); sptr > modp; sptr--) {
		if (*sptr == '/' && !--npath) {
		    sptr++;
		    break;
		}
	    }
	    if (*sptr == '/' && sptr[1] && sptr != modp)
		sptr++;
	    stradd(sptr);
	} else {
	    char cbu;
	    for (sptr = modp+1; *sptr; sptr++)
		if (*sptr == '/' && !++npath)
		    break;
	    cbu = *sptr;
	    *sptr = 0;
	    stradd(modp);
	    *sptr = cbu;
	}
    } else
	stradd(modp);

    if (p != modp)
	zsfree(modp);
}


match_colour(const char **teststrp, int is_fg, int colour)
{
    int shft, named = 0, tc;
    zattr on;

    if (is_fg) {
	shft = TXT_ATTR_FG_COL_SHIFT;
	on = TXTFGCOLOUR;
	tc = TCFGCOLOUR;
    } else {
	shft = TXT_ATTR_BG_COL_SHIFT;
	on = TXTBGCOLOUR;
	tc = TCBGCOLOUR;
    }
    if (teststrp) {
	if (**teststrp == '#' && isxdigit((*teststrp)[1])) {
	    struct color_rgb color;
	    char *end;
	    zlong col = zstrtol(*teststrp+1, &end, 16);
            if (end - *teststrp == 4) {
		color.red = col >> 8 | ((col >> 8) << 4);
		color.green = (col & 0xf0) >> 4;
		color.green |= color.green << 4;
		color.blue = col & 0xf;
		color.blue |= color.blue << 4;
	    } else if (end - *teststrp == 7) {
		color.red = col >> 16;
		color.green = (col & 0xff00) >> 8;
		color.blue = col & 0xff;
	    } else
		return TXT_ERROR;
	    *teststrp = end;
	    colour = runhookdef(GETCOLORATTR, &color) - 1;
	    if (colour == -1) { /* no hook function added, try true color (24-bit) */
		colour = (((color.red << 8) + color.green) << 8) + color.blue;
		return on | (is_fg ? TXT_ATTR_FG_24BIT : TXT_ATTR_BG_24BIT) |
			(zattr)colour << shft;
	    } else if (colour <= -2) {
		return TXT_ERROR;
	    }
	} else if ((named = ialpha(**teststrp))) {
	    colour = match_named_colour(teststrp);
	    if (colour == 8) {
		/* default */
		return is_fg ? TXTNOFGCOLOUR : TXTNOBGCOLOUR;
	    }
	    if (colour < 0)
		return TXT_ERROR;
	}
	else {
	    colour = (int)zstrtol(*teststrp, (char **)teststrp, 10);
	    if (colour < 0 || colour >= 256)
		return TXT_ERROR;
	}
    }
    /*
     * Try termcap for numbered characters if possible.
     * Don't for named characters, since our best bet
     * of getting the names right is with ANSI sequences.
     */
    if (!named && tccan(tc)) {
	if (tccolours >= 0 && colour >= tccolours) {
	    /*
	     * Out of range of termcap colours.
	     * Can we assume ANSI colours work?
	     */
	    if (colour > 7)
		return TXT_ERROR; /* No. */
	} else {
	    /*
	     * We can handle termcap colours and the number
	     * is in range, so use termcap.
	     */
	    on |= is_fg ? TXT_ATTR_FG_TERMCAP :
		TXT_ATTR_BG_TERMCAP;
	}
    }
    return on | (zattr)colour << shft;
}


parsecolorchar(zattr arg, int is_fg)
{
    if (bv->fm[1] == '{') {
	char *ep;
	bv->fm += 2; /* skip over F{ */
	if ((ep = strchr(bv->fm, '}'))) {
	    char oc = *ep, *col, *coll;
	    int ops = opts[PROMPTSUBST], opb = opts[PROMPTBANG];
	    int opp = opts[PROMPTPERCENT];

	    opts[PROMPTPERCENT] = 1;
	    opts[PROMPTSUBST] = opts[PROMPTBANG] = 0;

	    *ep = '\0';
	    /* expand the contents of the argument so you can use
	     * %v for example */
	    coll = col = promptexpand(bv->fm, 0, NULL, NULL, NULL);
	    *ep = oc;
	    arg = match_colour((const char **)&coll, is_fg, 0);
	    free(col);
	    bv->fm = ep;

	    opts[PROMPTSUBST] = ops;
	    opts[PROMPTBANG] = opb;
	    opts[PROMPTPERCENT] = opp;
	} else {
	    arg = match_colour((const char **)&bv->fm, is_fg, 0);
	    if (*bv->fm != '}')
		bv->fm--;
	}
    } else
	arg = match_colour(NULL, 1, arg);
    return arg;
}


putpromptchar(int doprint, int endchar, zattr *txtchangep)
{
    char *ss, *hostnam;
    int t0, arg, test, sep, j, numjobs, len;
    zattr atr;
    struct tm *tm;
    struct timespec ts;
    time_t timet;
    Nameddir nd;

    for (; *bv->fm && *bv->fm != endchar; bv->fm++) {
	arg = 0;
	if (*bv->fm == '%' && isset(PROMPTPERCENT)) {
	    int minus = 0;
	    bv->fm++;
	    if (*bv->fm == '-') {
		minus = 1;
		bv->fm++;
	    }
	    if (idigit(*bv->fm)) {
		arg = zstrtol(bv->fm, &bv->fm, 10);
		if (minus)
		    arg *= -1;
	    } else if (minus)
		arg = -1;
	    if (*bv->fm == '(') {
		int tc, otruncwidth;

		if (idigit(*++bv->fm)) {
		    arg = zstrtol(bv->fm, &bv->fm, 10);
		} else if (arg < 0) {
		    /* negative numbers don't make sense here */
		    arg *= -1;
		}
		test = 0;
		ss = pwd;
		switch (tc = *bv->fm) {
		case 'c':
		case '.':
		case '~':
		    if ((nd = finddir(ss))) {
			arg--;
			ss += strlen(nd->dir);
		    } /*FALLTHROUGH*/
		case '/':
		case 'C':
		    /* `/' gives 0, `/any' gives 1, etc. */
		    if (*ss && *ss++ == '/' && *ss)
			arg--;
		    for (; *ss; ss++)
			if (*ss == '/')
			    arg--;
		    if (arg <= 0)
			test = 1;
		    break;
		case 't':
		case 'T':
		case 'd':
		case 'D':
		case 'w':
		    timet = time(NULL);
		    tm = localtime(&timet);
		    switch (tc) {
		    case 't':
			test = (arg == tm->tm_min);
			break;
		    case 'T':
			test = (arg == tm->tm_hour);
			break;
		    case 'd':
			test = (arg == tm->tm_mday);
			break;
		    case 'D':
			test = (arg == tm->tm_mon);
			break;
		    case 'w':
			test = (arg == tm->tm_wday);
			break;
		    }
		    break;
		case '?':
		    if (lastval == arg)
			test = 1;
		    break;
		case '#':
		    if (geteuid() == (uid_t)arg)
			test = 1;
		    break;
		case 'g':
		    if (getegid() == (gid_t)arg)
			test = 1;
		    break;
		case 'j':
		    for (numjobs = 0, j = 1; j <= maxjob; j++)
			if (jobtab[j].stat && jobtab[j].procs &&
		    	    !(jobtab[j].stat & STAT_NOPRINT)) numjobs++;
		    if (numjobs >= arg)
		    	test = 1;
		    break;
		case 'l':
		    *bv->bp = '\0';
		    countprompt(bv->bufline, &t0, 0, 0);
		    if (minus)
			t0 = zterm_columns - t0;
		    if (t0 >= arg)
			test = 1;
		    break;
		case 'e':
		    {
			Funcstack fsptr = funcstack;
			test = arg;
			while (fsptr && test > 0) {
			    test--;
			    fsptr = fsptr->prev;
			}
			test = !test;
		    }
		    break;
		case 'L':
		    if (shlvl >= arg)
			test = 1;
		    break;
		case 'S':
		    if (time(NULL) - shtimer.tv_sec >= arg)
			test = 1;
		    break;
		case 'v':
		    if (arrlen_ge(psvar, arg))
			test = 1;
		    break;
		case 'V':
		    if (psvar && *psvar && arrlen_ge(psvar, arg)) {
			if (*psvar[(arg ? arg : 1) - 1])
			    test = 1;
		    }
		    break;
		case '_':
		    test = (cmdsp >= arg);
		    break;
		case '!':
		    test = privasserted();
		    break;
		default:
		    test = -1;
		    break;
		}
		if (!*bv->fm || !(sep = *++bv->fm))
		    return 0;
		bv->fm++;
		/* Don't do the current truncation until we get back */
		otruncwidth = bv->truncwidth;
		bv->truncwidth = 0;
		if (!putpromptchar(test == 1 && doprint, sep,
				   txtchangep) || !*++bv->fm ||
		    !putpromptchar(test == 0 && doprint, ')',
				   txtchangep)) {
		    bv->truncwidth = otruncwidth;
		    return 0;
		}
		bv->truncwidth = otruncwidth;
		continue;
	    }
	    if (!doprint)
		switch(*bv->fm) {
		  case '[':
		    while(idigit(*++bv->fm));
		    while(*++bv->fm != ']');
		    continue;
		  case '<':
		    while(*++bv->fm != '<');
		    continue;
		  case '>':
		    while(*++bv->fm != '>');
		    continue;
		  case 'D':
		    if(bv->fm[1]=='{')
			while(*++bv->fm != '}');
		    continue;
		  default:
		    continue;
		}
	    switch (*bv->fm) {
	    case '~':
		promptpath(pwd, arg, 1);
		break;
	    case 'd':
	    case '/':
		promptpath(pwd, arg, 0);
		break;
	    case 'c':
	    case '.':
		promptpath(pwd, arg ? arg : 1, 1);
		break;
	    case 'C':
		promptpath(pwd, arg ? arg : 1, 0);
		break;
	    case 'N':
		promptpath(scriptname ? scriptname : argzero, arg, 0);
		break;
	    case 'h':
	    case '!':
		addbufspc(DIGBUFSIZE);
		convbase(bv->bp, curhist, 10);
		bv->bp += strlen(bv->bp);
		break;
	    case 'j':
		for (numjobs = 0, j = 1; j <= maxjob; j++)
		    if (jobtab[j].stat && jobtab[j].procs &&
		    	!(jobtab[j].stat & STAT_NOPRINT)) numjobs++;
		addbufspc(DIGBUFSIZE);
		sprintf(bv->bp, "%d", numjobs);
		bv->bp += strlen(bv->bp);
		break;
	    case 'M':
		queue_signals();
		if ((hostnam = getsparam("HOST")))
		    stradd(hostnam);
		unqueue_signals();
		break;
	    case 'm':
		if (!arg)
		    arg++;
		queue_signals();
		if (!(hostnam = getsparam("HOST"))) {
		    unqueue_signals();
		    break;
		}
		if (arg < 0) {
		    for (ss = hostnam + strlen(hostnam); ss > hostnam; ss--)
			if (ss[-1] == '.' && !++arg)
			    break;
		    stradd(ss);
		} else {
		    for (ss = hostnam; *ss; ss++)
			if (*ss == '.' && !--arg)
			    break;
		    stradd(*ss ? dupstrpfx(hostnam, ss - hostnam) : hostnam);
		}
		unqueue_signals();
		break;
	    case 'S':
		txtchangeset(txtchangep, TXTSTANDOUT, TXTNOSTANDOUT);
		txtset(TXTSTANDOUT);
		tsetcap(TCSTANDOUTBEG, TSC_PROMPT);
		break;
	    case 's':
		txtchangeset(txtchangep, TXTNOSTANDOUT, TXTSTANDOUT);
		txtunset(TXTSTANDOUT);
		tsetcap(TCSTANDOUTEND, TSC_PROMPT|TSC_DIRTY);
		break;
	    case 'B':
		txtchangeset(txtchangep, TXTBOLDFACE, TXTNOBOLDFACE);
		txtset(TXTBOLDFACE);
		tsetcap(TCBOLDFACEBEG, TSC_PROMPT|TSC_DIRTY);
		break;
	    case 'b':
		txtchangeset(txtchangep, TXTNOBOLDFACE, TXTBOLDFACE);
		txtunset(TXTBOLDFACE);
		tsetcap(TCALLATTRSOFF, TSC_PROMPT|TSC_DIRTY);
		break;
	    case 'U':
		txtchangeset(txtchangep, TXTUNDERLINE, TXTNOUNDERLINE);
		txtset(TXTUNDERLINE);
		tsetcap(TCUNDERLINEBEG, TSC_PROMPT);
		break;
	    case 'u':
		txtchangeset(txtchangep, TXTNOUNDERLINE, TXTUNDERLINE);
		txtunset(TXTUNDERLINE);
		tsetcap(TCUNDERLINEEND, TSC_PROMPT|TSC_DIRTY);
		break;
	    case 'F':
		atr = parsecolorchar(arg, 1);
		if (!(atr & (TXT_ERROR | TXTNOFGCOLOUR))) {
		    txtchangeset(txtchangep, atr & TXT_ATTR_FG_ON_MASK,
				 TXTNOFGCOLOUR | TXT_ATTR_FG_COL_MASK);
		    txtunset(TXT_ATTR_FG_COL_MASK);
		    txtset(atr & TXT_ATTR_FG_ON_MASK);
		    set_colour_attribute(atr, COL_SEQ_FG, TSC_PROMPT);
		    break;
		}
		/* else FALLTHROUGH */
	    case 'f':
		txtchangeset(txtchangep, TXTNOFGCOLOUR, TXT_ATTR_FG_ON_MASK);
		txtunset(TXT_ATTR_FG_ON_MASK);
		set_colour_attribute(TXTNOFGCOLOUR, COL_SEQ_FG, TSC_PROMPT);
		break;
	    case 'K':
		atr = parsecolorchar(arg, 0);
		if (!(atr & (TXT_ERROR | TXTNOBGCOLOUR))) {
		    txtchangeset(txtchangep, atr & TXT_ATTR_BG_ON_MASK,
				 TXTNOBGCOLOUR | TXT_ATTR_BG_COL_MASK);
		    txtunset(TXT_ATTR_BG_COL_MASK);
		    txtset(atr & TXT_ATTR_BG_ON_MASK);
		    set_colour_attribute(atr, COL_SEQ_BG, TSC_PROMPT);
		    break;
		}
		/* else FALLTHROUGH */
	    case 'k':
		txtchangeset(txtchangep, TXTNOBGCOLOUR, TXT_ATTR_BG_ON_MASK);
		txtunset(TXT_ATTR_BG_ON_MASK);
		set_colour_attribute(TXTNOBGCOLOUR, COL_SEQ_BG, TSC_PROMPT);
		break;
	    case '[':
		if (idigit(*++bv->fm))
		    arg = zstrtol(bv->fm, &bv->fm, 10);
		if (!prompttrunc(arg, ']', doprint, endchar, txtchangep))
		    return *bv->fm;
		break;
	    case '<':
	    case '>':
		/* Test (minus) here so -0 means "at the right margin" */
		if (minus) {
		    *bv->bp = '\0';
		    countprompt(bv->bufline, &t0, 0, 0);
		    arg = zterm_columns - t0 + arg;
		    if (arg <= 0)
			arg = 1;
		}
		if (!prompttrunc(arg, *bv->fm, doprint, endchar, txtchangep))
		    return *bv->fm;
		break;
	    case '{': /*}*/
		if (!bv->dontcount++) {
		    addbufspc(1);
		    *bv->bp++ = Inpar;
		}
		if (arg <= 0)
		    break;
		/* else */
		/* FALLTHROUGH */
	    case 'G':
		if (arg > 0) {
		    addbufspc(arg);
		    while (arg--)
			*bv->bp++ = Nularg;
		} else {
		    addbufspc(1);
		    *bv->bp++ = Nularg;
		}
		break;
	    case /*{*/ '}':
		if (bv->trunccount && bv->trunccount >= bv->dontcount)
		    return *bv->fm;
		if (bv->dontcount && !--bv->dontcount) {
		    addbufspc(1);
		    *bv->bp++ = Outpar;
		}
		break;
	    case 't':
	    case '@':
	    case 'T':
	    case '*':
	    case 'w':
	    case 'W':
	    case 'D':
		{
		    char *tmfmt, *dd, *tmbuf = NULL;

		    switch (*bv->fm) {
		    case 'T':
			tmfmt = "%K:%M";
			break;
		    case '*':
			tmfmt = "%K:%M:%S";
			break;
		    case 'w':
			tmfmt = "%a %f";
			break;
		    case 'W':
			tmfmt = "%m/%d/%y";
			break;
		    case 'D':
			if (bv->fm[1] == '{' /*}*/) {
			    for (ss = bv->fm + 2; *ss && *ss != /*{*/ '}'; ss++)
				if(*ss == '\\' && ss[1])
				    ss++;
			    dd = tmfmt = tmbuf = zalloc(ss - bv->fm);
			    for (ss = bv->fm + 2; *ss && *ss != /*{*/ '}';
				 ss++) {
				if(*ss == '\\' && ss[1])
				    ss++;
				*dd++ = *ss;
			    }
			    *dd = 0;
			    bv->fm = ss - !*ss;
			    if (!*tmfmt) {
				free(tmbuf);
				continue;
			    }
			} else
			    tmfmt = "%y-%m-%d";
			break;
		    default:
			tmfmt = "%l:%M%p";
			break;
		    }
		    zgettime(&ts);
		    tm = localtime(&ts.tv_sec);
		    /*
		     * Hack because strftime won't say how
		     * much space it actually needs.  Try to add it
		     * a few times until it works.  Some formats don't
		     * actually have a length, so we could go on for
		     * ever.
		     */
		    for(j = 0, t0 = strlen(tmfmt)*8; j < 3; j++, t0*=2) {
			addbufspc(t0);
			if ((len = ztrftime(bv->bp, t0, tmfmt, tm, ts.tv_nsec))
			    >= 0)
			    break;
		    }
		    /* There is enough room for this because addbufspc(t0)
		     * allocates room for t0 * 2 bytes. */
		    if (len >= 0)
			metafy(bv->bp, len, META_NOALLOC);
		    bv->bp += strlen(bv->bp);
		    zsfree(tmbuf);
		    break;
		}
	    case 'n':
		stradd(get_username());
		break;
	    case 'l':
		if (*ttystrname) {
                   ss = (strncmp(ttystrname, "/dev/tty", 8) ?
                           ttystrname + 5 : ttystrname + 8);
		    stradd(ss);
		} else
		    stradd("()");
		break;
	    case 'y':
		if (*ttystrname) {
		    ss = (strncmp(ttystrname, "/dev/", 5) ?
			    ttystrname : ttystrname + 5);
		    stradd(ss);
		} else
		    stradd("()");
		break;
	    case 'L':
		addbufspc(DIGBUFSIZE);
#if defined(ZLONG_IS_LONG_LONG) && defined(PRINTF_HAS_LLD)
		sprintf(bv->bp, "%lld", shlvl);
#else
		sprintf(bv->bp, "%ld", (long)shlvl);
#endif
		bv->bp += strlen(bv->bp);
		break;
	    case '?':
		addbufspc(DIGBUFSIZE);
#if defined(ZLONG_IS_LONG_LONG) && defined(PRINTF_HAS_LLD)
		sprintf(bv->bp, "%lld", lastval);
#else
		sprintf(bv->bp, "%ld", (long)lastval);
#endif
		bv->bp += strlen(bv->bp);
		break;
	    case '%':
	    case ')':
		addbufspc(1);
		*bv->bp++ = *bv->fm;
		break;
	    case '#':
		addbufspc(1);
		*bv->bp++ = privasserted() ? '#' : '%';
		break;
	    case 'v':
		if (!arg)
		    arg = 1;
		else if (arg < 0)
		    arg += arrlen(psvar) + 1;
		if (arg > 0 && arrlen_ge(psvar, arg))
		    stradd(psvar[arg - 1]);
		break;
	    case 'E':
                tsetcap(TCCLEAREOL, TSC_PROMPT);
		break;
	    case '^':
		if (cmdsp) {
		    if (arg >= 0) {
			if (arg > cmdsp || arg == 0)
			    arg = cmdsp;
			for (t0 = cmdsp - 1; arg--; t0--) {
			    stradd(cmdnames[cmdstack[t0]]);
			    if (arg) {
				addbufspc(1);
				*bv->bp++=' ';
			    }
			}
		    } else {
			arg = -arg;
			if (arg > cmdsp)
			    arg = cmdsp;
			for (t0 = arg - 1; arg--; t0--) {
			    stradd(cmdnames[cmdstack[t0]]);
			    if (arg) {
				addbufspc(1);
				*bv->bp++=' ';
			    }
			}
		    }
		}
		break;
	    case '_':
		if (cmdsp) {
		    if (arg >= 0) {
			if (arg > cmdsp || arg == 0)
			    arg = cmdsp;
			for (t0 = cmdsp - arg; arg--; t0++) {
			    stradd(cmdnames[cmdstack[t0]]);
			    if (arg) {
				addbufspc(1);
				*bv->bp++=' ';
			    }
			}
		    } else {
			arg = -arg;
			if (arg > cmdsp)
			    arg = cmdsp;
			for (t0 = 0; arg--; t0++) {
			    stradd(cmdnames[cmdstack[t0]]);
			    if (arg) {
				addbufspc(1);
				*bv->bp++=' ';
			    }
			}
		    }
		}
		break;
	    case 'r':
		if(bv->rstring)
		    stradd(bv->rstring);
		break;
	    case 'R':
		if(bv->Rstring)
		    stradd(bv->Rstring);
		break;
	    case 'e':
	    {
		int depth = 0;
		Funcstack fsptr = funcstack;
		while (fsptr) {
		    depth++;
		    fsptr = fsptr->prev;
		}
		addbufspc(DIGBUFSIZE);
		sprintf(bv->bp, "%d", depth);
		bv->bp += strlen(bv->bp);
		break;
	    }
	    case 'I':
		if (funcstack && funcstack->tp != FS_SOURCE &&
		    !IN_EVAL_TRAP()) {
		    /*
		     * We're in a function or an eval with
		     * EVALLINENO.  Calculate the line number in
		     * the file.
		     */
		    zlong flineno = lineno + funcstack->flineno;
		    /* take account of eval line nos. starting at 1 */
		    if (funcstack->tp == FS_EVAL)
			lineno--;
		    addbufspc(DIGBUFSIZE);
#if defined(ZLONG_IS_LONG_LONG) && defined(PRINTF_HAS_LLD)
		    sprintf(bv->bp, "%lld", flineno);
#else
		    sprintf(bv->bp, "%ld", (long)flineno);
#endif
		    bv->bp += strlen(bv->bp);
		    break;
		}
		/* else we're in a file and lineno is already correct */
		/* FALLTHROUGH */
	    case 'i':
		addbufspc(DIGBUFSIZE);
#if defined(ZLONG_IS_LONG_LONG) && defined(PRINTF_HAS_LLD)
		sprintf(bv->bp, "%lld", lineno);
#else
		sprintf(bv->bp, "%ld", (long)lineno);
#endif
		bv->bp += strlen(bv->bp);
		break;
	    case 'x':
		if (funcstack && funcstack->tp != FS_SOURCE &&
		    !IN_EVAL_TRAP())
		    promptpath(funcstack->filename ? funcstack->filename : "",
			       arg, 0);
		else
		    promptpath(scriptfilename ? scriptfilename : argzero,
			       arg, 0);
		break;
	    case '\0':
		return 0;
	    case Meta:
		bv->fm++;
		break;
	    }
	} else if(*bv->fm == '!' && isset(PROMPTBANG)) {
	    if(doprint) {
		if(bv->fm[1] == '!') {
		    bv->fm++;
		    addbufspc(1);
		    pputc('!');
		} else {
		    addbufspc(DIGBUFSIZE);
		    convbase(bv->bp, curhist, 10);
		    bv->bp += strlen(bv->bp);
		}
	    }
	} else {
	    char c = *bv->fm == Meta ? *++bv->fm ^ 32 : *bv->fm;

	    if (doprint) {
		addbufspc(1);
		pputc(c);
	    }
	}
    }

    return *bv->fm;
}


addbufspc(int need)
{
    need *= 2;   /* for metafication */
    if((bv->bp - bv->buf) + need > bv->bufspc) {
	int bo = bv->bp - bv->buf;
	int bo1 = bv->bp1 ? bv->bp1 - bv->buf : -1;
	ptrdiff_t bufline_off = bv->bufline ? bv->bufline - bv->buf : -1;

	if(need & 255)
	    need = (need | 255) + 1;
	bv->buf = realloc(bv->buf, bv->bufspc += need);
	memset(bv->buf + bv->bufspc - need, 0, need);
	bv->bp = bv->buf + bo;
	if(bo1 != -1)
	    bv->bp1 = bv->buf + bo1;
	if (bufline_off != -1)
	    bv->bufline = bv->buf + bufline_off;
    }
}


tsetcap(int cap, int flags)
{
    if (tccan(cap) && !isset(SINGLELINEZLE) &&
        !(termflags & (TERM_NOUP|TERM_BAD|TERM_UNKNOWN))) {
	switch (flags & TSC_OUTPUT_MASK) {
	case TSC_RAW:
	    tputs(tcstr[cap], 1, putraw);
	    break;
	case 0:
	default:
	    tputs(tcstr[cap], 1, putshout);
	    break;
	case TSC_PROMPT:
	    if (!bv->dontcount) {
		addbufspc(1);
		*bv->bp++ = Inpar;
	    }
	    tputs(tcstr[cap], 1, putstr);
	    if (!bv->dontcount) {
		int glitch = 0;

		if (cap == TCSTANDOUTBEG || cap == TCSTANDOUTEND)
		    glitch = tgetnum("sg");
		else if (cap == TCUNDERLINEBEG || cap == TCUNDERLINEEND)
		    glitch = tgetnum("ug");
		if(glitch < 0)
		    glitch = 0;
		addbufspc(glitch + 1);
		while(glitch--)
		    *bv->bp++ = Nularg;
		*bv->bp++ = Outpar;
	    }
	    break;
	}

	if (flags & TSC_DIRTY) {
	    flags &= ~TSC_DIRTY;
	    if (txtisset(TXTBOLDFACE) && cap != TCBOLDFACEBEG)
		tsetcap(TCBOLDFACEBEG, flags);
	    if (txtisset(TXTSTANDOUT))
		tsetcap(TCSTANDOUTBEG, flags);
	    if (txtisset(TXTUNDERLINE))
		tsetcap(TCUNDERLINEBEG, flags);
	    if (txtisset(TXTFGCOLOUR))
		set_colour_attribute(txtattrmask, COL_SEQ_FG, TSC_PROMPT);
	    if (txtisset(TXTBGCOLOUR))
		set_colour_attribute(txtattrmask, COL_SEQ_BG, TSC_PROMPT);
	}
    }
}


cmdpop(void)
{
    if (cmdsp <= 0) {
	DPUTS(1, "BUG: cmdstack empty");
	fflush(stderr);
    } else
	cmdsp--;
}