Fixed issue where an index in qs_parse is incorrectly incremented beyond the maximum possible value

        std::vector<char*> get_list(const std::string& name, bool use_brackets = true) const
        {
            std::vector<char*> ret;
            std::string plus = name + (use_brackets ? "[]" : "");
            char* element = nullptr;

            int count = 0;
            while (1)
            {
                element = qs_k2v(plus.c_str(), key_value_pairs_.data(), key_value_pairs_.size(), count++);
                if (!element)
                    break;
                ret.push_back(element);
            }
            return ret;
        }


        std::unordered_map<std::string, std::string> pop_dict(const std::string& name)
        {
            std::unordered_map<std::string, std::string> ret = get_dict(name);
            if (!ret.empty())
            {
                for (unsigned int i = 0; i < key_value_pairs_.size(); i++)
                {
                    std::string str_item(key_value_pairs_[i]);
                    if (str_item.substr(0, name.size() + 1) == name + '[')
                    {
                        key_value_pairs_.erase(key_value_pairs_.begin() + i--);
                    }
                }
            }
            return ret;
        }


        std::vector<char*> pop_list(const std::string& name, bool use_brackets = true)
        {
            std::vector<char*> ret = get_list(name, use_brackets);
            if (!ret.empty())
            {
                for (unsigned int i = 0; i < key_value_pairs_.size(); i++)
                {
                    std::string str_item(key_value_pairs_[i]);
                    if ((use_brackets ? (str_item.substr(0, name.size() + 3) == name + "[]=") : (str_item.substr(0, name.size() + 1) == name + '=')))
                    {
                        key_value_pairs_.erase(key_value_pairs_.begin() + i--);
                    }
                }
            }
            return ret;
        }


        query_string& operator=(query_string&& qs)
        {
            key_value_pairs_ = std::move(qs.key_value_pairs_);
            char* old_data = (char*)qs.url_.c_str();
            url_ = std::move(qs.url_);
            for (auto& p : key_value_pairs_)
            {
                p += (char*)url_.c_str() - old_data;
            }
            return *this;
        }


inline int qs_decode(char * qs)
{
    int i=0, j=0;

    while( CROW_QS_ISQSCHR(qs[j]) )
    {
        if ( qs[j] == '+' ) {  qs[i] = ' ';  }
        else if ( qs[j] == '%' ) // easier/safer than scanf
        {
            if ( ! CROW_QS_ISHEX(qs[j+1]) || ! CROW_QS_ISHEX(qs[j+2]) )
            {
                qs[i] = '\0';
                return i;
            }
            qs[i] = (CROW_QS_HEX2DEC(qs[j+1]) * 16) + CROW_QS_HEX2DEC(qs[j+2]);
            j+=2;
        }
        else
        {
            qs[i] = qs[j];
        }
        i++;  j++;
    }
    qs[i] = '\0';

    return i;
}


        friend std::ostream& operator<<(std::ostream& os, const query_string& qs)
        {
            os << "[ ";
            for (size_t i = 0; i < qs.key_value_pairs_.size(); ++i)
            {
                if (i)
                    os << ", ";
                os << qs.key_value_pairs_[i];
            }
            os << " ]";
            return os;
        }


        query_string()
        {
        }


        std::unordered_map<std::string, std::string> get_dict(const std::string& name) const
        {
            std::unordered_map<std::string, std::string> ret;

            int count = 0;
            while (1)
            {
                if (auto element = qs_dict_name2kv(name.c_str(), key_value_pairs_.data(), key_value_pairs_.size(), count++))
                    ret.insert(*element);
                else
                    break;
            }
            return ret;
        }


inline int qs_parse(char* qs, char* qs_kv[], int qs_kv_size, bool parse_url = true)
{
    int i, j;
    char * substr_ptr;

    for(i=0; i<qs_kv_size; i++)  qs_kv[i] = NULL;

    // find the beginning of the k/v substrings or the fragment
    substr_ptr = parse_url ? qs + strcspn(qs, "?#") : qs;
    if (parse_url)
    {
        if (substr_ptr[0] != '\0')
            substr_ptr++;
        else
            return 0; // no query or fragment
    }

    i=0;
    while(i<qs_kv_size)
    {
        qs_kv[i] = substr_ptr;
        j = strcspn(substr_ptr, "&");
        if ( substr_ptr[j] == '\0' ) { i++; break;  } // x &'s -> means x iterations of this loop -> means *x+1* k/v pairs
        substr_ptr += j + 1;
        i++;
    }

    // we only decode the values in place, the keys could have '='s in them
    // which will hose our ability to distinguish keys from values later
    for(j=0; j<i; j++)
    {
        substr_ptr = qs_kv[j] + strcspn(qs_kv[j], "=&#");
        if ( substr_ptr[0] == '&' || substr_ptr[0] == '\0')  // blank value: skip decoding
            substr_ptr[0] = '\0';
        else
            qs_decode(++substr_ptr);
    }

#ifdef _qsSORTING
// TODO: qsort qs_kv, using qs_strncmp() for the comparison
#endif

    return i;
    }


        query_string(std::string params, bool url = true):
          url_(std::move(params))
        {
            if (url_.empty())
                return;

            key_value_pairs_.resize(MAX_KEY_VALUE_PAIRS_COUNT);

            int count = qs_parse(&url_[0], &key_value_pairs_[0], MAX_KEY_VALUE_PAIRS_COUNT, url);
            key_value_pairs_.resize(count);
        }


        char* pop(const std::string& name)
        {
            char* ret = get(name);
            if (ret != nullptr)
            {
                for (unsigned int i = 0; i < key_value_pairs_.size(); i++)
                {
                    std::string str_item(key_value_pairs_[i]);
                    if (str_item.substr(0, name.size() + 1) == name + '=')
                    {
                        key_value_pairs_.erase(key_value_pairs_.begin() + i);
                        break;
                    }
                }
            }
            return ret;
        }


        char* get(const std::string& name) const
        {
            char* ret = qs_k2v(name.c_str(), key_value_pairs_.data(), key_value_pairs_.size());
            return ret;
        }


        std::vector<std::string> keys() const
        {
            std::vector<std::string> ret;
            for (auto element : key_value_pairs_)
            {
                std::string str_element(element);
                ret.emplace_back(str_element.substr(0, str_element.find('=')));
            }
            return ret;
        }


inline char * qs_scanvalue(const char * key, const char * qs, char * val, size_t val_len)
{
    size_t i, key_len;
    const char * tmp;

    // find the beginning of the k/v substrings
    if ( (tmp = strchr(qs, '?')) != NULL )
        qs = tmp + 1;

    key_len = strlen(key);
    while(qs[0] != '#' && qs[0] != '\0')
    {
        if ( qs_strncmp(key, qs, key_len) == 0 )
            break;
        qs += strcspn(qs, "&") + 1;
    }

    if ( qs[0] == '\0' ) return NULL;

    qs += strcspn(qs, "=&#");
    if ( qs[0] == '=' )
    {
        qs++;
        i = strcspn(qs, "&=#");
#ifdef _MSC_VER
        strncpy_s(val, val_len, qs, (val_len - 1)<(i + 1) ? (val_len - 1) : (i + 1));
#else
        strncpy(val, qs, (val_len - 1)<(i + 1) ? (val_len - 1) : (i + 1));
#endif
		qs_decode(val);
    }
    else
    {
        if ( val_len > 0 )
            val[0] = '\0';
    }

    return val;
}


        void clear()
        {
            key_value_pairs_.clear();
            url_.clear();
        }


        query_string& operator=(const query_string& qs)
        {
            url_ = qs.url_;
            key_value_pairs_.clear();
            for (auto p : qs.key_value_pairs_)
            {
                key_value_pairs_.push_back((char*)(p - qs.url_.c_str() + url_.c_str()));
            }
            return *this;
        }


        query_string(const query_string& qs):
          url_(qs.url_)
        {
            for (auto p : qs.key_value_pairs_)
            {
                key_value_pairs_.push_back((char*)(p - qs.url_.c_str() + url_.c_str()));
            }
        }


inline char * qs_k2v(const char * key, char * const * qs_kv, int qs_kv_size, int nth = 0)
{
    int i;
    size_t key_len, skip;

    key_len = strlen(key);

#ifdef _qsSORTING
// TODO: binary search for key in the sorted qs_kv
#else  // _qsSORTING
    for(i=0; i<qs_kv_size; i++)
    {
        // we rely on the unambiguous '=' to find the value in our k/v pair
        if ( qs_strncmp(key, qs_kv[i], key_len) == 0 )
        {
            skip = strcspn(qs_kv[i], "=");
            if ( qs_kv[i][skip] == '=' )
                skip++;
            // return (zero-char value) ? ptr to trailing '\0' : ptr to value
            if(nth == 0)
                return qs_kv[i] + skip;
            else
                --nth;
        }
    }
#endif  // _qsSORTING

    return nullptr;
}


inline std::unique_ptr<std::pair<std::string, std::string>> qs_dict_name2kv(const char * dict_name, char * const * qs_kv, int qs_kv_size, int nth = 0)
{
    int i;
    size_t name_len, skip_to_eq, skip_to_brace_open, skip_to_brace_close;

    name_len = strlen(dict_name);

#ifdef _qsSORTING
// TODO: binary search for key in the sorted qs_kv
#else  // _qsSORTING
    for(i=0; i<qs_kv_size; i++)
    {
        if ( strncmp(dict_name, qs_kv[i], name_len) == 0 )
        {
            skip_to_eq = strcspn(qs_kv[i], "=");
            if ( qs_kv[i][skip_to_eq] == '=' )
                skip_to_eq++;
            skip_to_brace_open = strcspn(qs_kv[i], "[");
            if ( qs_kv[i][skip_to_brace_open] == '[' )
                skip_to_brace_open++;
            skip_to_brace_close = strcspn(qs_kv[i], "]");

            if ( skip_to_brace_open <= skip_to_brace_close &&
                 skip_to_brace_open > 0 &&
                 skip_to_brace_close > 0 &&
                 nth == 0 )
            {
                auto key = std::string(qs_kv[i] + skip_to_brace_open, skip_to_brace_close - skip_to_brace_open);
                auto value = std::string(qs_kv[i] + skip_to_eq);
                return std::unique_ptr<std::pair<std::string, std::string>>(new std::pair<std::string, std::string>(key, value));
            }
            else
            {
                --nth;
            }
        }
    }
#endif  // _qsSORTING

    return nullptr;
}


inline int qs_strncmp(const char * s, const char * qs, size_t n)
{
    int i=0;
    unsigned char u1, u2, unyb, lnyb;

    while(n-- > 0)
    {
        u1 = static_cast<unsigned char>(*s++);
        u2 = static_cast<unsigned char>(*qs++);

        if ( ! CROW_QS_ISQSCHR(u1) ) {  u1 = '\0';  }
        if ( ! CROW_QS_ISQSCHR(u2) ) {  u2 = '\0';  }

        if ( u1 == '+' ) {  u1 = ' ';  }
        if ( u1 == '%' ) // easier/safer than scanf
        {
            unyb = static_cast<unsigned char>(*s++);
            lnyb = static_cast<unsigned char>(*s++);
            if ( CROW_QS_ISHEX(unyb) && CROW_QS_ISHEX(lnyb) )
                u1 = (CROW_QS_HEX2DEC(unyb) * 16) + CROW_QS_HEX2DEC(lnyb);
            else
                u1 = '\0';
        }

        if ( u2 == '+' ) {  u2 = ' ';  }
        if ( u2 == '%' ) // easier/safer than scanf
        {
            unyb = static_cast<unsigned char>(*qs++);
            lnyb = static_cast<unsigned char>(*qs++);
            if ( CROW_QS_ISHEX(unyb) && CROW_QS_ISHEX(lnyb) )
                u2 = (CROW_QS_HEX2DEC(unyb) * 16) + CROW_QS_HEX2DEC(lnyb);
            else
                u2 = '\0';
        }

        if ( u1 != u2 )
            return u1 - u2;
        if ( u1 == '\0' )
            return 0;
        i++;
    }
    if ( CROW_QS_ISQSCHR(*qs) )
        return -1;
    else
        return 0;
}


Fixed typed-array ctor when source array is changed while iterating.

Previously, the function used optimization for ordinary arrays with no
gaps (so called fast arrays).  For a fast array code took elements
directly from internal flat C array. The direct pointer may become
invalid as side-effect of custom valueOf() method for an element.

The fix is to eliminate the micro-optimization which uses direct
pointers.

The problem is similar to the 9578cc729205 (0.7.2) commit.

This closes #523 issue on Github.

njs_typed_array_prototype_iterator_obj(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t kind)
{
    njs_value_t        *this;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    return njs_array_iterator_create(vm, this, &vm->retval, kind);
}


njs_typed_array_constructor_intrinsic(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    njs_type_error(vm, "Abstract class TypedArray not directly constructable");

    return NJS_ERROR;
}


njs_typed_array_compare(double a, double b)
{
    if (njs_slow_path(isnan(a))) {
        if (isnan(b)) {
            return 0;
        }

        return 1;
    }

    if (njs_slow_path(isnan(b))) {
        return -1;
    }

    if (a < b) {
        return -1;
    }

    if (a > b) {
        return 1;
    }

    return signbit(b) - signbit(a);
}


njs_typed_array_species_create(njs_vm_t *vm, njs_value_t *exemplar,
    njs_value_t *args, njs_uint_t nargs, njs_value_t *retval)
{
    njs_int_t          ret;
    njs_value_t        constructor;
    njs_typed_array_t  *array;

    array = njs_typed_array(exemplar);

    njs_set_function(&constructor, &vm->constructors[array->type]);

    ret = njs_value_species_constructor(vm, exemplar, &constructor,
                                        &constructor);
    if (njs_slow_path(ret != NJS_OK)) {
        return ret;
    }

    return njs_typed_array_create(vm, &constructor, args, nargs, retval);
}


njs_typed_array_get_i8(const void *a)
{
    return *(const int8_t *) a;
}


njs_typed_array_compare_f64(const void *a, const void *b, void *c)
{
    return njs_typed_array_compare(*(const double *) a, *(const double *) b);
}


njs_typed_array_of(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
    njs_index_t unused)
{
    double             num;
    uint32_t           length, i;
    njs_int_t          ret;
    njs_value_t        *this;
    njs_value_t        argument;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);

    if (njs_slow_path(!njs_is_constructor(this))) {
        njs_type_error(vm, "%s is not a constructor",
                       njs_type_string(this->type));
        return NJS_ERROR;
    }

    length = nargs - 1;

    njs_set_number(&argument, length);
    ret = njs_typed_array_create(vm, this, &argument, 1, &vm->retval);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    array = njs_typed_array(&vm->retval);

    for (i = 0; i < length; i++) {
        ret = njs_value_to_number(vm, njs_argument(args, i + 1), &num);
        if (njs_slow_path(ret != NJS_OK)) {
            return NJS_ERROR;
        }

        njs_typed_array_prop_set(vm, array, i, num);
    }

    njs_set_typed_array(&vm->retval, array);

    return NJS_OK;
}


njs_typed_array_prototype_iterator(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t type)
{
    double              val;
    int64_t             i, length;
    njs_int_t           ret;
    njs_arr_t           results;
    njs_value_t         *this, *this_arg, *r;
    njs_value_t         arguments[4], retval;
    njs_function_t      *function;
    njs_typed_array_t   *array, *dst;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    dst = NULL;
    array = njs_typed_array(this);
    length = njs_typed_array_length(array);

    if (njs_slow_path(!njs_is_function(njs_arg(args, nargs, 1)))) {
        njs_type_error(vm, "callback argument is not callable");
        return NJS_ERROR;
    }

    function = njs_function(njs_argument(args, 1));
    this_arg = njs_arg(args, nargs, 2);

    buffer = array->buffer;
    results.separate = 0;
    results.pointer = 0;

    switch (type) {
    case NJS_ARRAY_MAP:
        njs_set_number(&arguments[0], length);
        ret = njs_typed_array_species_create(vm, this, njs_value_arg(arguments),
                                             1, &retval);
        if (njs_slow_path(ret != NJS_OK)) {
            return NJS_ERROR;
        }

        dst = njs_typed_array(&retval);
        break;

    case NJS_ARRAY_FILTER:
    default:
        r = njs_arr_init(vm->mem_pool, &results, NULL, 4, sizeof(njs_value_t));
        if (njs_slow_path(r == NULL)) {
            return NJS_ERROR;
        }
    }

    for (i = 0; i < length; i++) {
        if (njs_slow_path(njs_is_detached_buffer(buffer))) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        val = njs_typed_array_prop(array, i);

        arguments[0] = *this_arg;
        njs_set_number(&arguments[1], val);
        njs_set_number(&arguments[2], i);
        njs_set_typed_array(&arguments[3], array);

        ret = njs_function_apply(vm, function, arguments, 4, &vm->retval);
        if (njs_slow_path(ret != NJS_OK)) {
            goto exception;
        }

        switch (type) {
        case NJS_ARRAY_EVERY:
            if (!njs_is_true(&vm->retval)) {
                vm->retval = njs_value_false;
                goto done;
            }

            break;

        case NJS_ARRAY_FOR_EACH:
            break;

        case NJS_ARRAY_SOME:
        case NJS_ARRAY_FIND:
        case NJS_ARRAY_FIND_INDEX:
            if (!njs_is_true(&vm->retval)) {
                continue;
            }

            switch (type) {
            case NJS_ARRAY_SOME:
                vm->retval = njs_value_true;
                break;

            case NJS_ARRAY_FIND:
                njs_set_number(&vm->retval, val);
                break;

            default:
                njs_set_number(&vm->retval, i);
                break;
            }

            goto done;

        case NJS_ARRAY_MAP:
            ret = njs_typed_array_set_value(vm, dst, i, &vm->retval);
            if (njs_slow_path(ret != NJS_OK)) {
                goto exception;
            }

            break;

        default:
            /* NJS_ARRAY_FILTER. */

            if (!njs_is_true(&vm->retval)) {
                continue;
            }

            r = njs_arr_add(&results);
            if (njs_slow_path(r == NULL)) {
                goto exception;
            }

            njs_set_number(r, val);
        }
    }

    /* Default values. */

    switch (type) {
    case NJS_ARRAY_EVERY:
        vm->retval = njs_value_true;
        break;

    case NJS_ARRAY_SOME:
        vm->retval = njs_value_false;
        break;

    case NJS_ARRAY_FOR_EACH:
    case NJS_ARRAY_FIND:
        njs_set_undefined(&vm->retval);
        break;

    case NJS_ARRAY_FIND_INDEX:
        njs_set_number(&vm->retval, -1);
        break;

    case NJS_ARRAY_MAP:
    case NJS_ARRAY_FILTER:
    default:
        if (type == NJS_ARRAY_FILTER) {
            njs_set_number(&arguments[0], results.items);
            ret = njs_typed_array_species_create(vm, this,
                                                 njs_value_arg(arguments),
                                                 1, &retval);
            if (njs_slow_path(ret != NJS_OK)) {
                goto exception;
            }

            dst = njs_typed_array(&retval);

            i = 0;

            while (i < results.items) {
                r = njs_arr_item(&results, i);
                ret = njs_typed_array_set_value(vm, dst, i++, r);
                if (njs_slow_path(ret != NJS_OK)) {
                    goto exception;
                }
            }
        }

        njs_set_typed_array(&vm->retval, dst);
        break;
    }

done:

    ret = NJS_OK;

exception:

    njs_arr_destroy(&results);

    return ret;
}


njs_typed_array_prototype_fill(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    float               f32;
    int8_t              i8;
    double              num;
    int16_t             i16;
    int32_t             i32;
    uint8_t             u8;
    int64_t             start, end, offset;
    uint32_t            i, length;
    njs_int_t           ret;
    njs_value_t         *this, *setval, lvalue;
    njs_typed_array_t   *array;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    length = njs_typed_array_length(array);

    setval = njs_lvalue_arg(&lvalue, args, nargs, 1);
    ret = njs_value_to_number(vm, setval, &num);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    ret = njs_value_to_integer(vm, njs_arg(args, nargs, 2), &start);
    if (njs_slow_path(ret != NJS_OK)) {
        return ret;
    }

    start = (start < 0) ? njs_max(length + start, 0) : njs_min(start, length);

    if (njs_is_undefined(njs_arg(args, nargs, 3))) {
        end = length;

    } else {
        ret = njs_value_to_integer(vm, njs_arg(args, nargs, 3), &end);
        if (njs_slow_path(ret != NJS_OK)) {
            return ret;
        }
    }

    end = (end < 0) ? njs_max(length + end, 0) : njs_min(end, length);

    buffer = njs_typed_array_writable(vm, array);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    njs_set_typed_array(&vm->retval, array);

    offset = array->offset;

    switch (array->type) {
    case NJS_OBJ_TYPE_UINT8_CLAMPED_ARRAY:
        if (isnan(num) || num < 0) {
            u8 = 0;

        } else if (num > 255) {
            u8 = 255;

        } else {
            u8 = lrint(num);
        }

        if (start < end) {
            memset(&buffer->u.u8[start + offset], u8, end - start);
        }

        break;

    case NJS_OBJ_TYPE_UINT8_ARRAY:
    case NJS_OBJ_TYPE_INT8_ARRAY:
        i8 = njs_number_to_int32(num);

        if (start < end) {
            memset(&buffer->u.u8[start + offset], i8, end - start);
        }

        break;

    case NJS_OBJ_TYPE_UINT16_ARRAY:
    case NJS_OBJ_TYPE_INT16_ARRAY:
        i16 = njs_number_to_int32(num);

        for (i = start; i < end; i++) {
            buffer->u.u16[i + offset] = i16;
        }

        break;

    case NJS_OBJ_TYPE_UINT32_ARRAY:
    case NJS_OBJ_TYPE_INT32_ARRAY:
        i32 = njs_number_to_int32(num);

        for (i = start; i < end; i++) {
            buffer->u.u32[i + offset] = i32;
        }

        break;

    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        f32 = num;

        for (i = start; i < end; i++) {
            buffer->u.f32[i + offset] = f32;
        }

        break;

    default:

        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        for (i = start; i < end; i++) {
            buffer->u.f64[i + offset] = num;
        }
    }

    return NJS_OK;
}


njs_typed_array_prototype_byte_length(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    size_t             byte_length;
    njs_value_t        *this;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);
    if (!njs_is_typed_array(this) && !njs_is_data_view(this)) {
        njs_type_error(vm, "Method TypedArray.prototype.byteLength called "
                       "on incompatible receiver");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    byte_length = array->byte_length;

    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        if (njs_is_data_view(this)) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        byte_length = 0;
    }

    njs_set_number(&vm->retval, byte_length);

    return NJS_OK;
}


njs_typed_array_compare_i32(const void *a, const void *b, void *c)
{
    int32_t  ai, bi;

    ai = *(const int32_t *) a;
    bi = *(const int32_t *) b;

    return (ai > bi) - (ai < bi);
}


njs_typed_array_prop_set(njs_vm_t *vm, njs_typed_array_t *array, uint32_t index,
    double v)
{
    int8_t              i8;
    int16_t             i16;
    int32_t             i32;
    njs_array_buffer_t  *buffer;

    buffer = array->buffer;
    index += array->offset;

    njs_assert(!buffer->object.shared);

    switch (array->type) {
    case NJS_OBJ_TYPE_UINT8_CLAMPED_ARRAY:
        if (isnan(v) || v < 0) {
            v = 0;
        } else if (v > 255) {
            v = 255;
        }

        buffer->u.u8[index] = lrint(v);

        break;

    case NJS_OBJ_TYPE_UINT8_ARRAY:
    case NJS_OBJ_TYPE_INT8_ARRAY:
        i8 = njs_number_to_int32(v);
        buffer->u.u8[index] = i8;
        break;

    case NJS_OBJ_TYPE_UINT16_ARRAY:
    case NJS_OBJ_TYPE_INT16_ARRAY:
        i16 = njs_number_to_int32(v);
        buffer->u.u16[index] = i16;
        break;

    case NJS_OBJ_TYPE_UINT32_ARRAY:
    case NJS_OBJ_TYPE_INT32_ARRAY:
        i32 = njs_number_to_int32(v);
        buffer->u.u32[index] = i32;
        break;

    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        buffer->u.f32[index] = v;
        break;

    default:

        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        buffer->u.f64[index] = v;
    }
}


njs_typed_array_compare_i16(const void *a, const void *b, void *c)
{
    return *((const int16_t *) a) - *((const int16_t *) b);
}


njs_typed_array_compare_i8(const void *a, const void *b, void *c)
{
    return *((const int8_t *) a) - *((const int8_t *) b);
}


njs_typed_array_prototype_byte_offset(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    size_t             byte_offset;
    njs_value_t        *this;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);
    if (!njs_is_typed_array(this) && !njs_is_data_view(this)) {
        njs_type_error(vm, "Method TypedArray.prototype.byteOffset called "
                       "on incompatible receiver");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    byte_offset = njs_typed_array_offset(array);

    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        if (njs_is_data_view(this)) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        byte_offset = 0;
    }

    njs_set_number(&vm->retval, byte_offset);

    return NJS_OK;
}


njs_typed_array_compare_f32(const void *a, const void *b, void *c)
{
    return njs_typed_array_compare(*(const float *) a, *(const float *) b);
}


njs_typed_array_prototype_join(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    u_char             *p;
    size_t             size, length, arr_length;
    njs_int_t          ret;
    njs_chb_t          chain;
    njs_value_t        *this, *separator;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    arr_length = njs_typed_array_length(array);

    separator = njs_arg(args, nargs, 1);

    if (njs_slow_path(!njs_is_string(separator))) {
        if (njs_is_undefined(separator)) {
            separator = njs_value_arg(&njs_string_comma);

        } else {
            ret = njs_value_to_string(vm, separator, separator);
            if (njs_slow_path(ret != NJS_OK)) {
                return ret;
            }
        }
    }

    if (arr_length == 0) {
        vm->retval = njs_string_empty;
        return NJS_OK;
    }

    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    njs_chb_init(&chain, vm->mem_pool);

    length = njs_typed_array_to_chain(vm, &chain, array, separator);
    size = njs_chb_size(&chain);

    p = njs_string_alloc(vm, &vm->retval, size, length);
    if (njs_slow_path(p == NULL)) {
        return NJS_ERROR;
    }

    njs_chb_join_to(&chain, p);
    njs_chb_destroy(&chain);

    return NJS_OK;
}


njs_typed_array_from(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
    njs_index_t unused)
{
    double             num;
    int64_t            length, i;
    njs_int_t          ret;
    njs_value_t        *this, *source, *mapfn;
    njs_value_t        arguments[3], retval;
    njs_function_t     *function;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);

    if (njs_slow_path(!njs_is_constructor(this))) {
        njs_type_error(vm, "%s is not a constructor",
                       njs_type_string(this->type));
        return NJS_ERROR;
    }

    mapfn = njs_arg(args, nargs, 2);

    if (njs_slow_path(!njs_is_function_or_undefined(mapfn))) {
        njs_type_error(vm, "\"mapfn\" argument is not callable");
        return NJS_ERROR;
    }

    function = NULL;
    if (njs_is_function(mapfn)) {
        function = njs_function(mapfn);
    }

    source = njs_arg(args, nargs, 1);

    ret = njs_value_to_object(vm, source);
    if (njs_slow_path(ret != NJS_OK)) {
        return ret;
    }

    ret = njs_object_length(vm, source, &length);
    if (njs_slow_path(ret == NJS_ERROR)) {
        return ret;
    }

    njs_set_number(&arguments[0], length);
    ret = njs_typed_array_create(vm, this, arguments, 1, &vm->retval);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    array = njs_typed_array(&vm->retval);
    arguments[0] = *njs_arg(args, nargs, 3);

    for (i = 0; i < length; i++) {
        ret = njs_value_property_i64(vm, source, i, &retval);
        if (njs_slow_path(ret == NJS_ERROR)) {
            return NJS_ERROR;
        }

        if (function != NULL) {
            arguments[1] = retval;
            njs_set_number(&arguments[2], i);
            ret = njs_function_apply(vm, function, arguments, 3, &retval);
            if (njs_slow_path(ret != NJS_OK)) {
                return NJS_ERROR;
            }
        }

        ret = njs_value_to_number(vm, &retval, &num);
        if (njs_slow_path(ret != NJS_OK)) {
            return NJS_ERROR;
        }

        njs_typed_array_prop_set(vm, array, i, num);
    }

    njs_set_typed_array(&vm->retval, array);

    return NJS_OK;
}


njs_typed_array_generic_compare(const void *a, const void *b, void *c)
{
    double                      num;
    njs_int_t                   ret;
    njs_value_t                 arguments[3], retval;
    njs_typed_array_sort_ctx_t  *ctx;

    ctx = c;

    if (njs_slow_path(ctx->exception)) {
        return 0;
    }

    njs_set_undefined(&arguments[0]);
    njs_set_number(&arguments[1], ctx->get(a));
    njs_set_number(&arguments[2], ctx->get(b));

    ret = njs_function_apply(ctx->vm, ctx->function, arguments, 3, &retval);
    if (njs_slow_path(ret != NJS_OK)) {
        goto exception;
    }

    ret = njs_value_to_number(ctx->vm, &retval, &num);
    if (njs_slow_path(ret != NJS_OK)) {
        goto exception;
    }

    if (njs_slow_path(njs_is_detached_buffer(ctx->buffer))) {
        njs_type_error(ctx->vm, "detached buffer");
        goto exception;
    }

    if (njs_slow_path(isnan(num))) {
        return 0;
    }

    if (num != 0) {
        return (num > 0) - (num < 0);
    }

    return 0;

exception:

    ctx->exception = 1;

    return 0;
}


njs_typed_array_writable(njs_vm_t *vm, njs_typed_array_t *array)
{
    njs_int_t           ret;
    njs_array_buffer_t  *buffer;

    buffer = array->buffer;
    if (njs_slow_path(njs_is_detached_buffer(buffer))) {
        njs_type_error(vm, "detached buffer");
        return NULL;
    }

    ret = njs_array_buffer_writable(vm, buffer);
    if (njs_slow_path(ret != NJS_OK)) {
        return NULL;
    }

    return buffer;
}


njs_data_view_prototype_set(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t type)
{
    double              v;
    uint8_t             *u8;
    uint32_t            u32;
    uint64_t            index;
    njs_int_t           ret;
    njs_bool_t          swap;
    njs_value_t         *this;
    njs_conv_f32_t      conv_f32;
    njs_conv_f64_t      conv_f64;
    njs_data_view_t     *view;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_data_view(this))) {
        njs_type_error(vm, "this is not a DataView");
        return NJS_ERROR;
    }

    ret = njs_value_to_index(vm, njs_arg(args, nargs, 1), &index);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    ret = njs_value_to_number(vm, njs_arg(args, nargs, 2), &v);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    swap = njs_bool(njs_arg(args, nargs, 3));

#if NJS_HAVE_LITTLE_ENDIAN
    swap = !swap;
#endif

    view = njs_data_view(this);
    if (njs_slow_path(njs_is_detached_buffer(view->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    if (njs_typed_array_element_size(type) + index > view->byte_length) {
        njs_range_error(vm, "index %uL is outside the bound of the buffer",
                        index);
        return NJS_ERROR;
    }

    buffer = view->buffer;
    u8 = &buffer->u.u8[index + view->offset];

    switch (type) {
    case NJS_OBJ_TYPE_UINT8_ARRAY:
    case NJS_OBJ_TYPE_INT8_ARRAY:
        *u8 = njs_number_to_int32(v);
        break;

    case NJS_OBJ_TYPE_UINT16_ARRAY:
    case NJS_OBJ_TYPE_INT16_ARRAY:
        u32 = (uint16_t) njs_number_to_int32(v);

        if (swap) {
            u32 = njs_bswap_u16(u32);
        }

        njs_set_u16(u8, u32);
        break;

    case NJS_OBJ_TYPE_UINT32_ARRAY:
    case NJS_OBJ_TYPE_INT32_ARRAY:
        u32 = njs_number_to_int32(v);

        if (swap) {
            u32 = njs_bswap_u32(u32);
        }

        njs_set_u32(u8, u32);
        break;

    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        conv_f32.f = (float) v;

        if (swap) {
            conv_f32.u = njs_bswap_u32(conv_f32.u);
        }

        njs_set_u32(u8, conv_f32.u);
        break;

    default:
        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        conv_f64.f = v;

        if (swap) {
            conv_f64.u = njs_bswap_u64(conv_f64.u);
        }

        njs_set_u64(u8, conv_f64.u);
    }

    njs_set_undefined(&vm->retval);

    return NJS_OK;
}


njs_typed_array_prototype_copy_within(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    int64_t             length, to, from, final, count;
    uint32_t            element_size;
    njs_int_t           ret;
    njs_value_t         *this, *value;
    njs_typed_array_t   *array;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    length = njs_typed_array_length(array);

    value = njs_arg(args, nargs, 1);

    ret = njs_value_to_integer(vm, value, &to);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    to = to < 0 ? njs_max(to + length, 0) : njs_min(to, length);

    value = njs_arg(args, nargs, 2);

    ret = njs_value_to_integer(vm, value, &from);
    if (njs_slow_path(ret != NJS_OK)) {
        return ret;
    }

    from = from < 0 ? njs_max(from + length, 0) : njs_min(from, length);

    value = njs_arg(args, nargs, 3);

    final = length;

    if (njs_is_defined(value)) {
        ret = njs_value_to_integer(vm, value, &final);
        if (njs_slow_path(ret != NJS_OK)) {
            return NJS_ERROR;
        }
    }

    final = (final < 0) ? njs_max(final + length, 0) : njs_min(final, length);

    njs_set_typed_array(&vm->retval, array);

    count = njs_min(final - from, length - to);

    if (count <= 0) {
        return NJS_OK;
    }

    buffer = njs_typed_array_writable(vm, array);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    element_size = njs_typed_array_element_size(array->type);

    to = (to + array->offset) * element_size;
    from = (from + array->offset) * element_size;
    count = count * element_size;

    memmove(&buffer->u.u8[to], &buffer->u.u8[from], count);

    return NJS_OK;
}


njs_typed_array_prototype_reverse(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    double              *f64;
    uint8_t             *u8;
    int64_t             i, length;
    uint16_t            *u16;
    uint32_t            *u32;
    njs_value_t         *this;
    njs_typed_array_t   *array;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    length = njs_typed_array_length(array);

    buffer = njs_typed_array_writable(vm, array);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    switch (array->type) {
    case NJS_OBJ_TYPE_UINT8_ARRAY:
    case NJS_OBJ_TYPE_UINT8_CLAMPED_ARRAY:
    case NJS_OBJ_TYPE_INT8_ARRAY:
        u8 = &buffer->u.u8[array->offset];

        for (i = 0; i < length / 2; i++) {
            njs_swap_u8(&u8[i], &u8[length - i - 1], 0);
        }

        break;

    case NJS_OBJ_TYPE_INT16_ARRAY:
    case NJS_OBJ_TYPE_UINT16_ARRAY:
        u16 = &buffer->u.u16[array->offset];

        for (i = 0; i < length / 2; i++) {
            njs_swap_u16(&u16[i], &u16[length - i - 1], 0);
        }

        break;

    case NJS_OBJ_TYPE_INT32_ARRAY:
    case NJS_OBJ_TYPE_UINT32_ARRAY:
    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        u32 = &buffer->u.u32[array->offset];

        for (i = 0; i < length / 2; i++) {
            njs_swap_u32(&u32[i], &u32[length - i - 1], 0);
        }

        break;

    default:

        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        f64 = &buffer->u.f64[array->offset];

        for (i = 0; i < length / 2; i++) {
            njs_swap_u64(&f64[i], &f64[length - i - 1], 0);
        }
    }

    njs_set_typed_array(&vm->retval, array);

    return NJS_OK;
}


njs_typed_array_get_i32(const void *a)
{
    return *(const int32_t *) a;
}


njs_typed_array_get_u32(const void *a)
{
    return *(const uint32_t *) a;
}


njs_typed_array_prototype_buffer(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    njs_value_t        *this;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);
    if (!njs_is_typed_array(this) && !njs_is_data_view(this)) {
        njs_type_error(vm, "Method TypedArray.prototype.buffer called "
                       "on incompatible receiver");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);

    njs_set_array_buffer(&vm->retval, njs_typed_array_buffer(array));

    return NJS_OK;
}


njs_typed_array_set_value(njs_vm_t *vm, njs_typed_array_t *array,
    uint32_t index, njs_value_t *setval)
{
    double              num;
    njs_int_t           ret;
    njs_array_buffer_t  *buffer;

    ret = njs_value_to_number(vm, setval, &num);
    if (njs_slow_path(ret != NJS_OK)) {
        return ret;
    }

    buffer = njs_typed_array_writable(vm, array);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    njs_typed_array_prop_set(vm, array, index, num);

    return NJS_OK;
}


njs_typed_array_get_this(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    vm->retval = args[0];

    return NJS_OK;
}


njs_typed_array_prototype_length(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    uint32_t           length;
    njs_value_t        *this;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);
    if (!njs_is_typed_array(this)) {
        njs_type_error(vm, "Method TypedArray.prototype.length called "
                       "on incompatible receiver");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    length = njs_typed_array_length(array);

    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        length = 0;
    }

    njs_set_number(&vm->retval, length);

    return NJS_OK;
}


njs_typed_array_prototype_set(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    double              num;
    int64_t             i, length, src_length, offset;
    njs_int_t           ret;
    njs_value_t         *this, *src, *value, prop;
    njs_array_t         *array;
    njs_typed_array_t   *self, *src_tarray;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    self = njs_typed_array(this);
    src = njs_arg(args, nargs, 1);
    value = njs_arg(args, nargs, 2);

    ret = njs_value_to_integer(vm, value, &offset);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    if (njs_slow_path(offset < 0)) {
        njs_range_error(vm, "offset is out of bounds");
        return NJS_ERROR;
    }

    buffer = njs_typed_array_writable(vm, self);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    length = njs_typed_array_length(self);

    if (njs_is_typed_array(src)) {
        src_tarray = njs_typed_array(src);
        if (njs_slow_path(njs_is_detached_buffer(src_tarray->buffer))) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        src_length = njs_typed_array_length(src_tarray);

        if (njs_slow_path((src_length > length)
                          || (offset > length - src_length)))
        {
            njs_range_error(vm, "source is too large");
            return NJS_ERROR;
        }

        length = njs_min(njs_typed_array_length(src_tarray), length - offset);

        for (i = 0; i < length; i++) {
            njs_typed_array_prop_set(vm, self, offset + i,
                                     njs_typed_array_prop(src_tarray, i));
        }

    } else {
        if (njs_is_fast_array(src)) {
            array = njs_array(src);
            src_length = array->length;

            if (njs_slow_path((src_length > length)
                              || (offset > length - src_length)))
            {
                njs_range_error(vm, "source is too large");
                return NJS_ERROR;
            }

            length = njs_min(array->length, length - offset);

            for (i = 0; i < length; i++) {
                ret = njs_value_to_number(vm, &array->start[i], &num);
                if (ret == NJS_OK) {
                    njs_typed_array_prop_set(vm, self, offset + i, num);
                }
            }

            goto done;
        }

        ret = njs_value_to_object(vm, src);
        if (njs_slow_path(ret != NJS_OK)) {
            return ret;
        }

        ret = njs_object_length(vm, src, &src_length);
        if (njs_slow_path(ret == NJS_ERROR)) {
            return ret;
        }

        if (njs_slow_path((src_length > length)
                          || (offset > length - src_length)))
        {
            njs_range_error(vm, "source is too large");
            return NJS_ERROR;
        }

        length = njs_min(src_length, length - offset);

        for (i = 0; i < length; i++) {
            ret = njs_value_property_i64(vm, src, i, &prop);
            if (njs_slow_path(ret == NJS_ERROR)) {
                return NJS_ERROR;
            }

            num = NAN;

            if (ret == NJS_OK) {
                ret = njs_value_to_number(vm, &prop, &num);
                if (njs_slow_path(ret == NJS_ERROR)) {
                    return NJS_ERROR;
                }
            }

            if (njs_slow_path(njs_is_detached_buffer(buffer))) {
                njs_type_error(vm, "detached buffer");
                return NJS_ERROR;
            }

            njs_typed_array_prop_set(vm, self, offset + i, num);
        }
    }

done:

    njs_set_undefined(&vm->retval);

    return NJS_OK;
}


njs_data_view_prototype_get(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t type)
{
    double              v;
    uint32_t            u32;
    uint64_t            index;
    njs_int_t           ret;
    njs_bool_t          swap;
    njs_value_t         *this;
    const uint8_t       *u8;
    njs_conv_f32_t      conv_f32;
    njs_conv_f64_t      conv_f64;
    njs_data_view_t     *view;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_data_view(this))) {
        njs_type_error(vm, "this is not a DataView");
        return NJS_ERROR;
    }

    ret = njs_value_to_index(vm, njs_arg(args, nargs, 1), &index);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    swap = njs_bool(njs_arg(args, nargs, 2));

#if NJS_HAVE_LITTLE_ENDIAN
    swap = !swap;
#endif

    view = njs_data_view(this);
    if (njs_slow_path(njs_is_detached_buffer(view->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    if (njs_typed_array_element_size(type) + index > view->byte_length) {
        njs_range_error(vm, "index %uL is outside the bound of the buffer",
                        index);
        return NJS_ERROR;
    }

    buffer = view->buffer;
    u8 = &buffer->u.u8[index + view->offset];

    switch (type) {
    case NJS_OBJ_TYPE_UINT8_ARRAY:
        v = *u8;
        break;

    case NJS_OBJ_TYPE_INT8_ARRAY:
        v = (int8_t) *u8;
        break;

    case NJS_OBJ_TYPE_UINT16_ARRAY:
        u32 = njs_get_u16(u8);

        if (swap) {
            u32 = njs_bswap_u16(u32);
        }

        v = u32;
        break;

    case NJS_OBJ_TYPE_INT16_ARRAY:
        u32 = njs_get_u16(u8);

        if (swap) {
            u32 = njs_bswap_u16(u32);
        }

        v = (int16_t) u32;
        break;

    case NJS_OBJ_TYPE_UINT32_ARRAY:
    case NJS_OBJ_TYPE_INT32_ARRAY:
    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        u32 = njs_get_u32(u8);

        if (swap) {
            u32 = njs_bswap_u32(u32);
        }

        switch (type) {
        case NJS_OBJ_TYPE_UINT32_ARRAY:
            v = u32;
            break;

        case NJS_OBJ_TYPE_INT32_ARRAY:
            v = (int32_t) u32;
            break;

        default:
            conv_f32.u = u32;
            v = conv_f32.f;
        }

        break;

    default:
        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        conv_f64.u = njs_get_u64(u8);

        if (swap) {
            conv_f64.u = njs_bswap_u64(conv_f64.u);
        }

        v = conv_f64.f;
    }

    njs_set_number(&vm->retval, v);

    return NJS_OK;
}


njs_typed_array_to_chain(njs_vm_t *vm, njs_chb_t *chain,
    njs_typed_array_t *array, njs_value_t *sep)
{
    size_t             size, length, arr_length;
    uint32_t           i;
    njs_string_prop_t  separator;

    if (sep == NULL) {
        sep = njs_value_arg(&njs_string_comma);
    }

    (void) njs_string_prop(&separator, sep);

    arr_length = njs_typed_array_length(array);

    if (arr_length == 0) {
        return 0;
    }

    for (i = 0; i < arr_length; i++) {
        njs_number_to_chain(vm, chain, njs_typed_array_prop(array, i));
        njs_chb_append(chain, separator.start, separator.size);
    }

    njs_chb_drop(chain, separator.size);

    size = njs_chb_size(chain);

    if (njs_utf8_length(separator.start, separator.size) >= 0) {
        length = size - (separator.size - separator.length) * (arr_length - 1);

    } else {
        length = 0;
    }

    return length;
}


njs_typed_array_get_u8(const void *a)
{
    return *(const uint8_t *) a;
}


njs_typed_array_get_f32(const void *a)
{
    return *(const float *) a;
}


njs_data_view_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
    njs_index_t unused)
{
    uint64_t            size, offset;
    njs_int_t           ret;
    njs_data_view_t     *view;
    njs_array_buffer_t  *buffer;

    if (!vm->top_frame->ctor) {
        njs_type_error(vm, "Constructor of DataView requires 'new'");
        return NJS_ERROR;
    }

    if (!njs_is_array_buffer(njs_arg(args, nargs, 1))) {
        njs_type_error(vm, "buffer is not an ArrayBuffer");
        return NJS_ERROR;
    }

    size = 0;
    offset = 0;

    ret = njs_value_to_index(vm, njs_arg(args, nargs, 2), &offset);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    buffer = njs_array_buffer(njs_argument(args, 1));
    if (njs_slow_path(njs_is_detached_buffer(buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    if (!njs_is_undefined(njs_arg(args, nargs, 3))) {
        ret = njs_value_to_index(vm, njs_argument(args, 3), &size);
        if (njs_slow_path(ret != NJS_OK)) {
            return NJS_ERROR;
        }

        if (njs_slow_path((offset + size) > buffer->size)) {
            njs_range_error(vm, "Invalid DataView length: %uL", size);
            return NJS_ERROR;
        }

    } else {
        if (offset > buffer->size) {
            njs_range_error(vm, "byteOffset %uL is outside the bound of "
                            "the buffer", offset);
            return NJS_ERROR;
        }

        size = buffer->size - offset;
    }

    view = njs_mp_zalloc(vm->mem_pool, sizeof(njs_data_view_t));
    if (njs_slow_path(view == NULL)) {
        goto memory_error;
    }

    view->buffer = buffer;
    view->offset = offset;
    view->byte_length = size;
    view->type = NJS_OBJ_TYPE_DATA_VIEW;

    njs_lvlhsh_init(&view->object.hash);
    njs_lvlhsh_init(&view->object.shared_hash);
    view->object.__proto__ = &vm->prototypes[view->type].object;
    view->object.type = NJS_DATA_VIEW;
    view->object.extensible = 1;

    njs_set_data_view(&vm->retval, view);

    return NJS_OK;

memory_error:

    njs_memory_error(vm);

    return NJS_ERROR;
}


njs_typed_array_prototype_index_of(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t type)
{
    double              v;
    int64_t             i, i64, from, to, index, increment, offset, length;
    njs_int_t           ret, integer;
    njs_value_t         *this;
    const float         *f32;
    const double        *f64;
    const uint8_t       *u8;
    const uint16_t      *u16;
    const uint32_t      *u32;
    njs_typed_array_t   *array;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    index = -1;
    array = njs_typed_array(this);
    length = njs_typed_array_length(array);

    if (!njs_is_number(njs_arg(args, nargs, 1)) || length == 0) {
        goto done;
    }

    if (type & 2) {
        /* lastIndexOf(). */

        if (nargs > 2) {
            ret = njs_value_to_integer(vm, njs_arg(args, nargs, 2), &from);
            if (njs_slow_path(ret != NJS_OK)) {
                return ret;
            }

        } else {
            from = length - 1;
        }

        if (from >= 0) {
            from = njs_min(from, length - 1);

        } else if (from < 0) {
            from += length;
        }

        to = -1;
        increment = -1;

        if (from <= to) {
            goto done;
        }

    } else {
        /* indexOf(), includes(). */

        ret = njs_value_to_integer(vm, njs_arg(args, nargs, 2), &from);
        if (njs_slow_path(ret != NJS_OK)) {
            return ret;
        }

        if (from < 0) {
            from += length;

            if (from < 0) {
                from = 0;
            }
        }

        to = length;
        increment = 1;

        if (from >= to) {
            goto done;
        }
    }

    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    v = njs_number(njs_argument(args, 1));

    i64 = v;
    integer = (v == i64);

    buffer = array->buffer;
    offset = array->offset;

    switch (array->type) {
    case NJS_OBJ_TYPE_INT8_ARRAY:
        if (integer && ((int8_t) i64 == i64)) {
            goto search8;
        }

        break;

    case NJS_OBJ_TYPE_UINT8_CLAMPED_ARRAY:
    case NJS_OBJ_TYPE_UINT8_ARRAY:
        if (integer && ((uint8_t) i64 == i64)) {
search8:
            u8 = &buffer->u.u8[0];
            for (i = from; i != to; i += increment) {
                if (u8[offset + i] == (uint8_t) i64) {
                    index = i;
                    break;
                }
            }
        }

        break;

    case NJS_OBJ_TYPE_INT16_ARRAY:
        if (integer && ((int16_t) i64 == i64)) {
            goto search16;
        }

        break;

    case NJS_OBJ_TYPE_UINT16_ARRAY:
        if (integer && ((uint16_t) i64 == i64)) {
search16:
            u16 = &buffer->u.u16[0];
            for (i = from; i != to; i += increment) {
                if (u16[offset + i] == (uint16_t) i64) {
                    index = i;
                    break;
                }
            }
        }

        break;

    case NJS_OBJ_TYPE_INT32_ARRAY:
        if (integer && ((int32_t) i64 == i64)) {
            goto search32;
        }

        break;

    case NJS_OBJ_TYPE_UINT32_ARRAY:
        if (integer && ((uint32_t) i64 == i64)) {
search32:
            u32 = &buffer->u.u32[0];
            for (i = from; i != to; i += increment) {
                if (u32[offset + i] == (uint32_t) i64) {
                    index = i;
                    break;
                }
            }
        }

        break;

    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        f32 = &buffer->u.f32[0];

        if (((float) v == v)) {
            for (i = from; i != to; i += increment) {
                if (f32[offset + i] == (float) v) {
                    index = i;
                    break;
                }
            }

        } else if ((type & 1) && isnan(v)) {
            /* includes() handles NaN. */

            for (i = from; i != to; i += increment) {
                if (isnan(f32[offset + i])) {
                    index = i;
                    break;
                }
            }
        }

        break;

    default:

        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        f64 = &buffer->u.f64[0];

        if ((type & 1) && isnan(v)) {
            /* includes() handles NaN. */

            for (i = from; i != to; i += increment) {
                if (isnan(f64[offset + i])) {
                    index = i;
                    break;
                }
            }

        } else {
            for (i = from; i != to; i += increment) {
                if (f64[offset + i] == v) {
                    index = i;
                    break;
                }
            }
        }
    }

done:

    /* Default values. */

    if (type & 1) {
        njs_set_boolean(&vm->retval, index != -1);

    } else {
        njs_set_number(&vm->retval, index);
    }

    return NJS_OK;
}


njs_typed_array_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
    njs_index_t magic)
{
    njs_typed_array_t  *array;

    if (!vm->top_frame->ctor) {
        njs_type_error(vm, "Constructor of TypedArray requires 'new'");
        return NJS_ERROR;
    }

    array = njs_typed_array_alloc(vm, &args[1], nargs - 1, 1, magic);
    if (njs_slow_path(array == NULL)) {
        return NJS_ERROR;
    }

    njs_set_typed_array(&vm->retval, array);

    return NJS_OK;
}


njs_typed_array_prototype_reduce(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t right)
{
    int64_t             i, from, to, increment, length;
    njs_int_t           ret;
    njs_value_t         *this, accumulator;
    njs_value_t         arguments[5];
    njs_function_t      *function;
    njs_typed_array_t   *array;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    length = njs_typed_array_length(array);

    if (njs_slow_path(!njs_is_function(njs_arg(args, nargs, 1)))) {
        njs_type_error(vm, "callback argument is not callable");
        return NJS_ERROR;
    }

    function = njs_function(njs_argument(args, 1));

    if (length == 0 && nargs <= 2) {
        njs_type_error(vm, "Reduce of empty object with no initial value");
        return NJS_ERROR;
    }

    if (right) {
        from = length - 1;
        to = -1;
        increment = -1;

    } else {
        from = 0;
        to = length;
        increment = 1;
    }

    buffer = array->buffer;

    if (nargs > 2) {
        accumulator = *njs_argument(args, 2);

    } else {
        if (njs_slow_path(njs_is_detached_buffer(buffer))) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        njs_set_number(&accumulator, njs_typed_array_prop(array, from));
        from += increment;
    }

    for (i = from; i != to; i += increment) {
        if (njs_slow_path(njs_is_detached_buffer(buffer))) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        njs_set_undefined(&arguments[0]);
        arguments[1] = accumulator;
        njs_set_number(&arguments[2], njs_typed_array_prop(array, i));
        njs_set_number(&arguments[3], i);
        njs_set_typed_array(&arguments[4], array);

        ret =  njs_function_apply(vm, function, arguments, 5, &accumulator);
        if (njs_slow_path(ret != NJS_OK)) {
            return ret;
        }
    }

    vm->retval = accumulator;

    return NJS_OK;
}


njs_typed_array_get_i16(const void *a)
{
    return *(const int16_t *) a;
}


njs_typed_array_alloc(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
    njs_bool_t zeroing, njs_object_type_t type)
{
    double              num;
    int64_t             i, length;
    uint32_t            element_size;
    uint64_t            size, offset;
    njs_int_t           ret;
    njs_value_t         *value, prop;
    njs_typed_array_t   *array, *src_tarray;
    njs_array_buffer_t  *buffer;

    size = 0;
    length = 0;
    offset = 0;

    buffer = NULL;
    src_tarray = NULL;

    element_size = njs_typed_array_element_size(type);

    value = njs_arg(args, nargs, 0);

    if (njs_is_array_buffer(value)) {
        buffer = njs_array_buffer(value);

        ret = njs_value_to_index(vm, njs_arg(args, nargs, 1), &offset);
        if (njs_slow_path(ret != NJS_OK)) {
            return NULL;
        }

        if (njs_slow_path((offset % element_size) != 0)) {
            njs_range_error(vm, "start offset must be multiple of %uD",
                            element_size);
            return NULL;
        }

        if (njs_is_defined(njs_arg(args, nargs, 2))) {
            ret = njs_value_to_index(vm, njs_argument(args, 2), &size);
            if (njs_slow_path(ret != NJS_OK)) {
                return NULL;
            }
        }

        if (njs_slow_path(njs_is_detached_buffer(buffer))) {
            njs_type_error(vm, "detached buffer");
            return NULL;
        }

        if (njs_is_defined(njs_arg(args, nargs, 2))) {
            ret = njs_value_to_index(vm, njs_argument(args, 2), &size);
            if (njs_slow_path(ret != NJS_OK)) {
                return NULL;
            }

            size *= element_size;

            if (njs_slow_path((offset + size) > buffer->size)) {
                njs_range_error(vm, "Invalid typed array length: %uL", size);
                return NULL;
            }

        } else {
            if (njs_slow_path((buffer->size % element_size) != 0)) {
                njs_range_error(vm, "byteLength of buffer must be "
                                "multiple of %uD", element_size);
                return NULL;
            }

            if (offset > buffer->size) {
                njs_range_error(vm, "byteOffset %uL is outside the bound of "
                                "the buffer", offset);
                return NULL;
            }

            size = buffer->size - offset;
        }

    } else if (njs_is_typed_array(value)) {
        src_tarray = njs_typed_array(value);
        if (njs_slow_path(njs_is_detached_buffer(src_tarray->buffer))) {
            njs_type_error(vm, "detached buffer");
            return NULL;
        }

        size = (uint64_t) njs_typed_array_length(src_tarray) * element_size;

    } else if (njs_is_object(value)) {
        ret = njs_object_length(vm, value, &length);
        if (njs_slow_path(ret == NJS_ERROR)) {
            return NULL;
        }

        size = length * element_size;

    } else {
        ret = njs_value_to_index(vm, value, &size);
        if (njs_slow_path(ret != NJS_OK)) {
            return NULL;
        }

        size *= element_size;
    }

    if (buffer == NULL) {
        buffer = njs_array_buffer_alloc(vm, size, zeroing);
        if (njs_slow_path(buffer == NULL)) {
            return NULL;
        }
    }

    array = njs_mp_zalloc(vm->mem_pool, sizeof(njs_typed_array_t));
    if (njs_slow_path(array == NULL)) {
        goto memory_error;
    }

    array->buffer = buffer;
    array->offset = offset / element_size;
    array->byte_length = size;
    array->type = type;

    if (src_tarray != NULL) {
        if (type != src_tarray->type) {
            length = njs_typed_array_length(src_tarray);
            for (i = 0; i < length; i++) {
                njs_typed_array_prop_set(vm, array, i,
                                         njs_typed_array_prop(src_tarray, i));
            }

        } else {
            memcpy(&buffer->u.u8[0], &src_tarray->buffer->u.u8[0], size);
        }

    } else if (!njs_is_array_buffer(value) && njs_is_object(value)) {
        for (i = 0; i < length; i++) {
            ret = njs_value_property_i64(vm, value, i, &prop);
            if (njs_slow_path(ret == NJS_ERROR)) {
                return NULL;
            }

            num = NAN;

            if (ret == NJS_OK) {
                ret = njs_value_to_number(vm, &prop, &num);
                if (njs_slow_path(ret == NJS_ERROR)) {
                    return NULL;
                }
            }

            njs_typed_array_prop_set(vm, array, i, num);
        }
    }

    njs_lvlhsh_init(&array->object.hash);
    njs_lvlhsh_init(&array->object.shared_hash);
    array->object.__proto__ = &vm->prototypes[type].object;
    array->object.type = NJS_TYPED_ARRAY;
    array->object.extensible = 1;
    array->object.fast_array = 1;

    return array;

memory_error:

    njs_memory_error(vm);

    return NULL;
}


njs_typed_array_compare_u16(const void *a, const void *b, void *c)
{
    return *((const uint16_t *) a) - *((const uint16_t *) b);
}


njs_typed_array_get_u16(const void *a)
{
    return *(const uint16_t *) a;
}


njs_typed_array_compare_u8(const void *a, const void *b, void *c)
{
    return *((const uint8_t *) a) - *((const uint8_t *) b);
}


njs_typed_array_create(njs_vm_t *vm, njs_value_t *constructor,
    njs_value_t *args, njs_uint_t nargs, njs_value_t *retval)
{
    njs_int_t          ret;
    njs_value_t        this;
    njs_object_t       *object;
    njs_typed_array_t  *array;

    object = njs_function_new_object(vm, constructor);
    if (njs_slow_path(object == NULL)) {
        return NJS_ERROR;
    }

    njs_set_object(&this, object);

    ret = njs_function_call2(vm, njs_function(constructor), &this, args,
                             nargs, retval, 1);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    if (njs_slow_path(!njs_is_typed_array(retval))) {
        njs_type_error(vm, "Derived TypedArray constructor "
                       "returned not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(retval);
    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    if (njs_slow_path(nargs == 1 && njs_is_number(&args[0])
                      && njs_typed_array_length(array)
                         < njs_number(&args[0])))
    {
        njs_type_error(vm, "Derived TypedArray constructor "
                       "returned too short array");
        return NJS_ERROR;
    }

    return NJS_OK;
}


njs_typed_array_get_string_tag(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    njs_value_t  *this;

    static const njs_value_t  *tags[NJS_OBJ_TYPE_TYPED_ARRAY_SIZE] = {
        &njs_typed_array_uint8_tag,
        &njs_typed_array_uint8_clamped_tag,
        &njs_typed_array_int8_tag,
        &njs_typed_array_uint16_tag,
        &njs_typed_array_int16_tag,
        &njs_typed_array_uint32_tag,
        &njs_typed_array_int32_tag,
        &njs_typed_array_float32_tag,
        &njs_typed_array_float64_tag,
    };

    this = njs_argument(args, 0);

    if (!njs_is_typed_array(this)) {
        njs_set_undefined(&vm->retval);
        return NJS_OK;
    }

    vm->retval = *tags[njs_typed_array_index(njs_typed_array(this)->type)];

    return NJS_OK;
}


njs_typed_array_prototype_slice(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t copy)
{
    int64_t             start, end, count, offset;
    uint32_t            i, element_size, length;
    njs_int_t           ret;
    njs_value_t         arguments[3], *this, *value;
    njs_typed_array_t   *array, *new_array;
    njs_array_buffer_t  *buffer, *new_buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    length = njs_typed_array_length(array);
    buffer = njs_typed_array_buffer(array);
    if (njs_slow_path(copy && njs_is_detached_buffer(buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    ret = njs_value_to_integer(vm, njs_arg(args, nargs, 1), &start);
    if (njs_slow_path(ret != NJS_OK)) {
        njs_range_error(vm, "invalid start");
        return NJS_ERROR;
    }

    start = (start < 0) ? njs_max(length + start, 0) : njs_min(start, length);

    value = njs_arg(args, nargs, 2);

    if (njs_is_undefined(value)) {
        end = length;

    } else {
        ret = njs_value_to_integer(vm, value, &end);
        if (njs_slow_path(ret != NJS_OK)) {
            njs_range_error(vm, "invalid end");
            return NJS_ERROR;
        }
    }

    end = (end < 0) ? njs_max(length + end, 0) : njs_min(end, length);

    element_size = njs_typed_array_element_size(array->type);

    if (copy) {
        count = njs_max(end - start, 0);
        njs_set_number(&arguments[0], count);

        ret = njs_typed_array_species_create(vm, this,
                                             njs_value_arg(arguments), 1,
                                             &vm->retval);
        if (njs_slow_path(ret != NJS_OK)) {
            return ret;
        }

        if (count == 0) {
            return NJS_OK;
        }

        if (njs_slow_path(njs_is_detached_buffer(buffer))) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        new_array = njs_typed_array(&vm->retval);
        new_buffer = njs_typed_array_buffer(new_array);
        element_size = njs_typed_array_element_size(array->type);

        if (njs_fast_path(array->type == new_array->type)) {
            start = start * element_size;
            count = count * element_size;

            memcpy(&new_buffer->u.u8[0], &buffer->u.u8[start], count);

        } else {
            for (i = 0; i < count; i++) {
                njs_typed_array_prop_set(vm, new_array, i,
                                        njs_typed_array_prop(array, i + start));
            }
        }

        return NJS_OK;
    }

    offset = array->offset * element_size;
    offset += start * element_size;

    njs_set_array_buffer(&arguments[0], buffer);
    njs_set_number(&arguments[1], offset);
    njs_set_number(&arguments[2], njs_max(end - start, 0));

    return njs_typed_array_species_create(vm, this, njs_value_arg(arguments), 3,
                                          &vm->retval);
}


njs_typed_array_compare_u32(const void *a, const void *b, void *c)
{
    uint32_t  au, bu;

    au = *(const uint32_t *) a;
    bu = *(const uint32_t *) b;

    return (au > bu) - (au < bu);
}


njs_typed_array_prototype_sort(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    u_char                      *base, *orig;
    int64_t                     length;
    uint32_t                    element_size;
    njs_value_t                 *this, *comparefn;
    njs_typed_array_t           *array;
    njs_array_buffer_t          *buffer;
    njs_typed_array_cmp_t       cmp;
    njs_typed_array_sort_ctx_t  ctx;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    ctx.vm = vm;
    ctx.buffer = array->buffer;
    ctx.exception = 0;

    comparefn = njs_arg(args, nargs, 1);

    if (njs_is_defined(comparefn)) {
        if (njs_slow_path(!njs_is_function(comparefn))) {
            njs_type_error(vm, "comparefn must be callable or undefined");
            return NJS_ERROR;
        }

        ctx.function = njs_function(comparefn);

    } else {
        ctx.function = NULL;
    }

    switch (array->type) {
    case NJS_OBJ_TYPE_UINT8_ARRAY:
    case NJS_OBJ_TYPE_UINT8_CLAMPED_ARRAY:
        cmp = njs_typed_array_compare_u8;
        ctx.get = njs_typed_array_get_u8;
        break;

    case NJS_OBJ_TYPE_INT8_ARRAY:
        cmp = njs_typed_array_compare_i8;
        ctx.get = njs_typed_array_get_i8;
        break;

    case NJS_OBJ_TYPE_UINT16_ARRAY:
        cmp = njs_typed_array_compare_u16;
        ctx.get = njs_typed_array_get_u16;
        break;

    case NJS_OBJ_TYPE_INT16_ARRAY:
        cmp = njs_typed_array_compare_i16;
        ctx.get = njs_typed_array_get_i16;
        break;

    case NJS_OBJ_TYPE_UINT32_ARRAY:
        cmp = njs_typed_array_compare_u32;
        ctx.get = njs_typed_array_get_u32;
        break;

    case NJS_OBJ_TYPE_INT32_ARRAY:
        cmp = njs_typed_array_compare_i32;
        ctx.get = njs_typed_array_get_i32;
        break;

    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        cmp = njs_typed_array_compare_f32;
        ctx.get = njs_typed_array_get_f32;
        break;

    default:

        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        cmp = njs_typed_array_compare_f64;
        ctx.get = njs_typed_array_get_f64;
        break;
    }

    buffer = njs_typed_array_writable(vm, array);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    length = njs_typed_array_length(array);
    element_size = njs_typed_array_element_size(array->type);
    base = &buffer->u.u8[array->offset * element_size];
    orig = base;

    if (ctx.function != NULL) {
        cmp = njs_typed_array_generic_compare;
        base = njs_mp_alloc(vm->mem_pool, length * element_size);
        if (njs_slow_path(base == NULL)) {
            njs_memory_error(vm);
            return NJS_ERROR;
        }

        memcpy(base, &buffer->u.u8[array->offset * element_size],
               length * element_size);
    }

    njs_qsort(base, length, element_size, cmp, &ctx);
    if (ctx.function != NULL) {
        if (&buffer->u.u8[array->offset * element_size] == orig) {
            memcpy(orig, base, length * element_size);
        }

        njs_mp_free(vm->mem_pool, base);
    }

    if (njs_slow_path(ctx.exception)) {
        return NJS_ERROR;
    }

    njs_set_typed_array(&vm->retval, array);

    return NJS_OK;
}


njs_typed_array_get_f64(const void *a)
{
    return *(const double *) a;
}


SERVER-60218-44: SERVER-60218 add initialize helper function for document_source_group (cherry picked from commit 867f52afbb79bc00e35c70f8e0681b7d602f97b2)

const char* DocumentSourceGroup::getSourceName() const {
    return kStageName.rawData();
}


intrusive_ptr<DocumentSourceGroup> DocumentSourceGroup::create(
    const intrusive_ptr<ExpressionContext>& pExpCtx,
    const boost::intrusive_ptr<Expression>& groupByExpression,
    std::vector<AccumulationStatement> accumulationStatements,
    boost::optional<size_t> maxMemoryUsageBytes) {
    size_t memoryBytes = maxMemoryUsageBytes ? *maxMemoryUsageBytes
                                             : internalDocumentSourceGroupMaxMemoryBytes.load();
    intrusive_ptr<DocumentSourceGroup> groupStage(new DocumentSourceGroup(pExpCtx, memoryBytes));
    groupStage->setIdExpression(groupByExpression);
    for (auto&& statement : accumulationStatements) {
        groupStage->addAccumulator(statement);
    }

    return groupStage;
}


    bool operator()(const GroupsMap::value_type* lhs, const GroupsMap::value_type* rhs) const {
        return _valueComparator.evaluate(lhs->first < rhs->first);
    }


int DocumentSourceGroup::freeMemory() {
    invariant(_groups);
    int totalMemorySaved = 0;
    for (auto&& group : *_groups) {
        for (auto&& groupObj : group.second) {
            auto prevMemUsage = groupObj->memUsageForSorter();
            groupObj->reduceMemoryConsumptionIfAble();

            // Update the memory usage for this group.
            totalMemorySaved += (prevMemUsage - groupObj->memUsageForSorter());
        }
    }
    return totalMemorySaved;
}


DocumentSource::GetNextResult DocumentSourceGroup::doGetNext() {
    if (!_initialized) {
        const auto initializationResult = initialize();
        if (initializationResult.isPaused()) {
            return initializationResult;
        }
        invariant(initializationResult.isEOF());
    }

    for (auto&& accum : _currentAccumulators) {
        accum->reset();  // Prep accumulators for a new group.
    }

    if (_spilled) {
        return getNextSpilled();
    } else {
        return getNextStandard();
    }
}


boost::optional<DocumentSource::DistributedPlanLogic> DocumentSourceGroup::distributedPlanLogic() {
    intrusive_ptr<DocumentSourceGroup> mergingGroup(new DocumentSourceGroup(pExpCtx));
    mergingGroup->setDoingMerge(true);

    VariablesParseState vps = pExpCtx->variablesParseState;
    /* the merger will use the same grouping key */
    mergingGroup->setIdExpression(ExpressionFieldPath::parse(pExpCtx, "$$ROOT._id", vps));

    for (auto&& accumulatedField : _accumulatedFields) {
        // The merger's output field names will be the same, as will the accumulator factories.
        // However, for some accumulators, the expression to be accumulated will be different. The
        // original accumulator may be collecting an expression based on a field expression or
        // constant.  Here, we accumulate the output of the same name from the prior group.
        auto copiedAccumulatedField = accumulatedField;
        copiedAccumulatedField.expr.argument =
            ExpressionFieldPath::parse(pExpCtx, "$$ROOT." + copiedAccumulatedField.fieldName, vps);
        mergingGroup->addAccumulator(copiedAccumulatedField);
    }

    // {shardsStage, mergingStage, sortPattern}
    return DistributedPlanLogic{this, mergingGroup, boost::none};
}


Document GroupFromFirstDocumentTransformation::serializeTransformation(
    boost::optional<ExplainOptions::Verbosity> explain) const {

    MutableDocument newRoot(_accumulatorExprs.size());
    for (auto&& expr : _accumulatorExprs) {
        newRoot.addField(expr.first, expr.second->serialize(static_cast<bool>(explain)));
    }

    return {{"newRoot", newRoot.freezeToValue()}};
}


Value DocumentSourceGroup::serialize(boost::optional<ExplainOptions::Verbosity> explain) const {
    MutableDocument insides;

    // Add the _id.
    if (_idFieldNames.empty()) {
        invariant(_idExpressions.size() == 1);
        insides["_id"] = _idExpressions[0]->serialize(static_cast<bool>(explain));
    } else {
        // Decomposed document case.
        invariant(_idExpressions.size() == _idFieldNames.size());
        MutableDocument md;
        for (size_t i = 0; i < _idExpressions.size(); i++) {
            md[_idFieldNames[i]] = _idExpressions[i]->serialize(static_cast<bool>(explain));
        }
        insides["_id"] = md.freezeToValue();
    }

    // Add the remaining fields.
    for (auto&& accumulatedField : _accumulatedFields) {
        intrusive_ptr<AccumulatorState> accum = accumulatedField.makeAccumulator();
        insides[accumulatedField.fieldName] =
            Value(accum->serialize(accumulatedField.expr.initializer,
                                   accumulatedField.expr.argument,
                                   static_cast<bool>(explain)));
    }

    if (_doingMerge) {
        // This makes the output unparsable (with error) on pre 2.6 shards, but it will never
        // be sent to old shards when this flag is true since they can't do a merge anyway.
        insides["$doingMerge"] = Value(true);
    }

    return Value(DOC(getSourceName() << insides.freeze()));
}


bool DocumentSourceGroup::canRunInParallelBeforeWriteStage(
    const std::set<std::string>& nameOfShardKeyFieldsUponEntryToStage) const {
    if (_doingMerge) {
        return true;  // This is fine.
    }

    // Certain $group stages are allowed to execute on each exchange consumer. In order to
    // guarantee each consumer will only group together data from its own shard, the $group must
    // group on a superset of the shard key.
    for (auto&& currentPathOfShardKey : nameOfShardKeyFieldsUponEntryToStage) {
        if (!pathIncludedInGroupKeys(currentPathOfShardKey)) {
            // This requires an exact path match, but as a future optimization certain path
            // prefixes should be okay. For example, if the shard key path is "a.b", and we're
            // grouping by "a", then each group of "a" is strictly more specific than "a.b", so
            // we can deduce that grouping by "a" will not need to group together documents
            // across different values of the shard key field "a.b", and thus as long as any
            // other shard key fields are similarly preserved will not need to consume a merged
            // stream to perform the group.
            return false;
        }
    }
    return true;
}


Value DocumentSourceGroup::expandId(const Value& val) {
    // _id doesn't get wrapped in a document
    if (_idFieldNames.empty())
        return val;

    // _id is a single-field document containing val
    if (_idFieldNames.size() == 1)
        return Value(DOC(_idFieldNames[0] << val));

    // _id is a multi-field document containing the elements of val
    const vector<Value>& vals = val.getArray();
    invariant(_idFieldNames.size() == vals.size());
    MutableDocument md(vals.size());
    for (size_t i = 0; i < vals.size(); i++) {
        md[_idFieldNames[i]] = vals[i];
    }
    return md.freezeToValue();
}


MONGO_COMPILER_NOINLINE DocumentSource::GetNextResult DocumentSourceGroup::initializeSelf(
    GetNextResult input) {
    const size_t numAccumulators = _accumulatedFields.size();
    // Barring any pausing, this loop exhausts 'pSource' and populates '_groups'.
    for (; input.isAdvanced(); input = pSource->getNext()) {
        if (_memoryTracker.shouldSpillWithAttemptToSaveMemory([this]() { return freeMemory(); })) {
            _sortedFiles.push_back(spill());
        }

        // We release the result document here so that it does not outlive the end of this loop
        // iteration. Not releasing could lead to an array copy when this group follows an unwind.
        auto rootDocument = input.releaseDocument();
        Value id = computeId(rootDocument);

        // Look for the _id value in the map. If it's not there, add a new entry with a blank
        // accumulator. This is done in a somewhat odd way in order to avoid hashing 'id' and
        // looking it up in '_groups' multiple times.
        const size_t oldSize = _groups->size();
        vector<intrusive_ptr<AccumulatorState>>& group = (*_groups)[id];
        const bool inserted = _groups->size() != oldSize;

        if (inserted) {
            _memoryTracker.memoryUsageBytes += id.getApproximateSize();

            // Initialize and add the accumulators
            Value expandedId = expandId(id);
            Document idDoc =
                expandedId.getType() == BSONType::Object ? expandedId.getDocument() : Document();
            group.reserve(numAccumulators);
            for (auto&& accumulatedField : _accumulatedFields) {
                auto accum = accumulatedField.makeAccumulator();
                Value initializerValue =
                    accumulatedField.expr.initializer->evaluate(idDoc, &pExpCtx->variables);
                accum->startNewGroup(initializerValue);
                group.push_back(accum);
            }
        } else {
            for (auto&& groupObj : group) {
                // subtract old mem usage. New usage added back after processing.
                _memoryTracker.memoryUsageBytes -= groupObj->memUsageForSorter();
            }
        }

        /* tickle all the accumulators for the group we found */
        dassert(numAccumulators == group.size());

        for (size_t i = 0; i < numAccumulators; i++) {
            group[i]->process(
                _accumulatedFields[i].expr.argument->evaluate(rootDocument, &pExpCtx->variables),
                _doingMerge);

            _memoryTracker.memoryUsageBytes += group[i]->memUsageForSorter();
        }

        if (kDebugBuild && !storageGlobalParams.readOnly) {
            // In debug mode, spill every time we have a duplicate id to stress merge logic.
            if (!inserted &&                     // is a dup
                !pExpCtx->inMongos &&            // can't spill to disk in mongos
                !_memoryTracker.allowDiskUse &&  // don't change behavior when testing external sort
                _sortedFiles.size() < 20) {      // don't open too many FDs

                _sortedFiles.push_back(spill());
            }
        }
    }

    switch (input.getStatus()) {
        case DocumentSource::GetNextResult::ReturnStatus::kAdvanced: {
            MONGO_UNREACHABLE;  // We consumed all advances above.
        }
        case DocumentSource::GetNextResult::ReturnStatus::kPauseExecution: {
            return input;  // Propagate pause.
        }
        case DocumentSource::GetNextResult::ReturnStatus::kEOF: {
            // Do any final steps necessary to prepare to output results.
            if (!_sortedFiles.empty()) {
                _spilled = true;
                if (!_groups->empty()) {
                    _sortedFiles.push_back(spill());
                }

                // We won't be using groups again so free its memory.
                _groups = pExpCtx->getValueComparator().makeUnorderedValueMap<Accumulators>();

                _sorterIterator.reset(Sorter<Value, Value>::Iterator::merge(
                    _sortedFiles, SortOptions(), SorterComparator(pExpCtx->getValueComparator())));

                // prepare current to accumulate data
                _currentAccumulators.reserve(numAccumulators);
                for (auto&& accumulatedField : _accumulatedFields) {
                    _currentAccumulators.push_back(accumulatedField.makeAccumulator());
                }

                verify(_sorterIterator->more());  // we put data in, we should get something out.
                _firstPartOfNextGroup = _sorterIterator->next();
            } else {
                // start the group iterator
                groupsIterator = _groups->begin();
            }

            // This must happen last so that, unless control gets here, we will re-enter
            // initialization after getting a GetNextResult::ResultState::kPauseExecution.
            _initialized = true;
            return input;
        }
    }
    MONGO_UNREACHABLE;
}


const std::vector<AccumulationStatement>& DocumentSourceGroup::getAccumulatedFields() const {
    return _accumulatedFields;
}


bool DocumentSourceGroup::pathIncludedInGroupKeys(const std::string& dottedPath) const {
    return std::any_of(
        _idExpressions.begin(), _idExpressions.end(), [&dottedPath](const auto& exp) {
            if (auto fieldExp = dynamic_cast<ExpressionFieldPath*>(exp.get())) {
                if (fieldExp->representsPath(dottedPath)) {
                    return true;
                }
            }
            return false;
        });
}


void DocumentSourceGroup::addAccumulator(AccumulationStatement accumulationStatement) {
    _accumulatedFields.push_back(accumulationStatement);
}


bool DocumentSourceGroup::MemoryUsageTracker::shouldSpillWithAttemptToSaveMemory(
    std::function<int()> saveMemory) {
    if (!allowDiskUse && (memoryUsageBytes > maxMemoryUsageBytes)) {
        memoryUsageBytes -= saveMemory();
    }

    if (memoryUsageBytes > maxMemoryUsageBytes) {
        uassert(ErrorCodes::QueryExceededMemoryLimitNoDiskUseAllowed,
                "Exceeded memory limit for $group, but didn't allow external sort."
                " Pass allowDiskUse:true to opt in.",
                allowDiskUse);
        memoryUsageBytes = 0;
        return true;
    }
    return false;
}


SpillSTLComparator(ValueComparator valueComparator) : _valueComparator(valueComparator) {}


Document DocumentSourceGroup::makeDocument(const Value& id,
                                           const Accumulators& accums,
                                           bool mergeableOutput) {
    const size_t n = _accumulatedFields.size();
    MutableDocument out(1 + n);

    /* add the _id field */
    out.addField("_id", expandId(id));

    /* add the rest of the fields */
    for (size_t i = 0; i < n; ++i) {
        Value val = accums[i]->getValue(mergeableOutput);
        if (val.missing()) {
            // we return null in this case so return objects are predictable
            out.addField(_accumulatedFields[i].fieldName, Value(BSONNULL));
        } else {
            out.addField(_accumulatedFields[i].fieldName, val);
        }
    }

    return out.freeze();
}


StringMap<boost::intrusive_ptr<Expression>> DocumentSourceGroup::getIdFields() const {
    if (_idFieldNames.empty()) {
        invariant(_idExpressions.size() == 1);
        return {{"_id", _idExpressions[0]}};
    } else {
        invariant(_idFieldNames.size() == _idExpressions.size());
        StringMap<boost::intrusive_ptr<Expression>> result;
        for (std::size_t i = 0; i < _idFieldNames.size(); ++i) {
            result["_id." + _idFieldNames[i]] = _idExpressions[i];
        }
        return result;
    }
}


DepsTracker::State DocumentSourceGroup::getDependencies(DepsTracker* deps) const {
    // add the _id
    for (size_t i = 0; i < _idExpressions.size(); i++) {
        _idExpressions[i]->addDependencies(deps);
    }

    // add the rest
    for (auto&& accumulatedField : _accumulatedFields) {
        accumulatedField.expr.argument->addDependencies(deps);
        // Don't add initializer, because it doesn't refer to docs from the input stream.
    }

    return DepsTracker::State::EXHAUSTIVE_ALL;
}


DocumentSource::GetModPathsReturn DocumentSourceGroup::getModifiedPaths() const {
    // We preserve none of the fields, but any fields referenced as part of the group key are
    // logically just renamed.
    StringMap<std::string> renames;
    for (std::size_t i = 0; i < _idExpressions.size(); ++i) {
        auto idExp = _idExpressions[i];
        auto pathToPutResultOfExpression =
            _idFieldNames.empty() ? "_id" : "_id." + _idFieldNames[i];
        auto computedPaths = idExp->getComputedPaths(pathToPutResultOfExpression);
        for (auto&& rename : computedPaths.renames) {
            renames[rename.first] = rename.second;
        }
    }

    return {DocumentSource::GetModPathsReturn::Type::kAllExcept,
            std::set<std::string>{},  // No fields are preserved.
            std::move(renames)};
}


void DocumentSourceGroup::doDispose() {
    // Free our resources.
    _groups = pExpCtx->getValueComparator().makeUnorderedValueMap<Accumulators>();
    _sorterIterator.reset();

    // Make us look done.
    groupsIterator = _groups->end();
}


DepsTracker::State GroupFromFirstDocumentTransformation::addDependencies(DepsTracker* deps) const {
    for (auto&& expr : _accumulatorExprs) {
        expr.second->addDependencies(deps);
    }

    // This stage will replace the entire document with a new document, so any existing fields
    // will be replaced and cannot be required as dependencies. We use EXHAUSTIVE_ALL here
    // instead of EXHAUSTIVE_FIELDS, as in ReplaceRootTransformation, because the stages that
    // follow a $group stage should not depend on document metadata.
    return DepsTracker::State::EXHAUSTIVE_ALL;
}


std::string nextFileName() {
    static AtomicWord<unsigned> documentSourceGroupFileCounter;
    return "extsort-doc-group." + std::to_string(documentSourceGroupFileCounter.fetchAndAdd(1));
}


intrusive_ptr<DocumentSource> DocumentSourceGroup::createFromBson(
    BSONElement elem, const intrusive_ptr<ExpressionContext>& pExpCtx) {
    uassert(15947, "a group's fields must be specified in an object", elem.type() == Object);

    intrusive_ptr<DocumentSourceGroup> pGroup(new DocumentSourceGroup(pExpCtx));

    BSONObj groupObj(elem.Obj());
    BSONObjIterator groupIterator(groupObj);
    VariablesParseState vps = pExpCtx->variablesParseState;
    while (groupIterator.more()) {
        BSONElement groupField(groupIterator.next());
        StringData pFieldName = groupField.fieldNameStringData();
        if (pFieldName == "_id") {
            uassert(
                15948, "a group's _id may only be specified once", pGroup->_idExpressions.empty());
            pGroup->setIdExpression(parseIdExpression(pExpCtx, groupField, vps));
            invariant(!pGroup->_idExpressions.empty());
        } else if (pFieldName == "$doingMerge") {
            massert(17030, "$doingMerge should be true if present", groupField.Bool());

            pGroup->setDoingMerge(true);
        } else {
            // Any other field will be treated as an accumulator specification.
            pGroup->addAccumulator(
                AccumulationStatement::parseAccumulationStatement(pExpCtx, groupField, vps));
        }
    }

    uassert(15955, "a group specification must include an _id", !pGroup->_idExpressions.empty());
    return pGroup;
}


DocumentSource::GetNextResult DocumentSourceGroup::getNextStandard() {
    // Not spilled, and not streaming.
    if (_groups->empty())
        return GetNextResult::makeEOF();

    Document out = makeDocument(groupsIterator->first, groupsIterator->second, pExpCtx->needsMerge);

    if (++groupsIterator == _groups->end())
        dispose();

    return std::move(out);
}


    int operator()(const Data& lhs, const Data& rhs) const {
        return _valueComparator.compare(lhs.first, rhs.first);
    }


bool DocumentSourceGroup::usedDisk() {
    return _usedDisk;
}


intrusive_ptr<Expression> parseIdExpression(const intrusive_ptr<ExpressionContext>& expCtx,
                                            BSONElement groupField,
                                            const VariablesParseState& vps) {
    if (groupField.type() == Object) {
        // {_id: {}} is treated as grouping on a constant, not an expression
        if (groupField.Obj().isEmpty()) {
            return ExpressionConstant::create(expCtx, Value(groupField));
        }

        const BSONObj idKeyObj = groupField.Obj();
        if (idKeyObj.firstElementFieldName()[0] == '$') {
            // grouping on a $op expression
            return Expression::parseObject(expCtx, idKeyObj, vps);
        } else {
            for (auto&& field : idKeyObj) {
                uassert(17390,
                        "$group does not support inclusion-style expressions",
                        !field.isNumber() && field.type() != Bool);
            }
            return ExpressionObject::parse(expCtx, idKeyObj, vps);
        }
    } else {
        return Expression::parseOperand(expCtx, groupField, vps);
    }
}


shared_ptr<Sorter<Value, Value>::Iterator> DocumentSourceGroup::spill() {
    _usedDisk = true;
    vector<const GroupsMap::value_type*> ptrs;  // using pointers to speed sorting
    ptrs.reserve(_groups->size());
    for (GroupsMap::const_iterator it = _groups->begin(), end = _groups->end(); it != end; ++it) {
        ptrs.push_back(&*it);
    }

    stable_sort(ptrs.begin(), ptrs.end(), SpillSTLComparator(pExpCtx->getValueComparator()));

    SortedFileWriter<Value, Value> writer(SortOptions().TempDir(pExpCtx->tempDir), _file);
    switch (_accumulatedFields.size()) {  // same as ptrs[i]->second.size() for all i.
        case 0:                           // no values, essentially a distinct
            for (size_t i = 0; i < ptrs.size(); i++) {
                writer.addAlreadySorted(ptrs[i]->first, Value());
            }
            break;

        case 1:  // just one value, use optimized serialization as single Value
            for (size_t i = 0; i < ptrs.size(); i++) {
                writer.addAlreadySorted(ptrs[i]->first,
                                        ptrs[i]->second[0]->getValue(/*toBeMerged=*/true));
            }
            break;

        default:  // multiple values, serialize as array-typed Value
            for (size_t i = 0; i < ptrs.size(); i++) {
                vector<Value> accums;
                for (size_t j = 0; j < ptrs[i]->second.size(); j++) {
                    accums.push_back(ptrs[i]->second[j]->getValue(/*toBeMerged=*/true));
                }
                writer.addAlreadySorted(ptrs[i]->first, Value(std::move(accums)));
            }
            break;
    }

    _groups->clear();

    Sorter<Value, Value>::Iterator* iteratorPtr = writer.done();
    return shared_ptr<Sorter<Value, Value>::Iterator>(iteratorPtr);
}


Value DocumentSourceGroup::computeId(const Document& root) {
    // If only one expression, return result directly
    if (_idExpressions.size() == 1) {
        Value retValue = _idExpressions[0]->evaluate(root, &pExpCtx->variables);
        return retValue.missing() ? Value(BSONNULL) : std::move(retValue);
    }

    // Multiple expressions get results wrapped in a vector
    vector<Value> vals;
    vals.reserve(_idExpressions.size());
    for (size_t i = 0; i < _idExpressions.size(); i++) {
        vals.push_back(_idExpressions[i]->evaluate(root, &pExpCtx->variables));
    }
    return Value(std::move(vals));
}


DocumentSourceGroup::DocumentSourceGroup(const intrusive_ptr<ExpressionContext>& pExpCtx,
                                         boost::optional<size_t> maxMemoryUsageBytes)
    : DocumentSource(kStageName, pExpCtx),
      _usedDisk(false),
      _doingMerge(false),
      _memoryTracker{pExpCtx->allowDiskUse && !pExpCtx->inMongos,
                     maxMemoryUsageBytes ? *maxMemoryUsageBytes
                                         : internalDocumentSourceGroupMaxMemoryBytes.load()},
      // We spill to disk in debug mode, regardless of allowDiskUse, to stress the system.
      _file(!pExpCtx->inMongos && (pExpCtx->allowDiskUse || kDebugBuild)
                ? std::make_shared<Sorter<Value, Value>::File>(pExpCtx->tempDir + "/" +
                                                               nextFileName())
                : nullptr),
      _initialized(false),
      _groups(pExpCtx->getValueComparator().makeUnorderedValueMap<Accumulators>()),
      _spilled(false) {}


DocumentSource::GetModPathsReturn GroupFromFirstDocumentTransformation::getModifiedPaths() const {
    // Replaces the entire root, so all paths are modified.
    return {DocumentSource::GetModPathsReturn::Type::kAllPaths, std::set<std::string>{}, {}};
}


DocumentSource::GetNextResult DocumentSourceGroup::initialize() {
    GetNextResult input = pSource->getNext();
    return initializeSelf(input);
}


DocumentSourceGroup::rewriteGroupAsTransformOnFirstDocument() const {
    if (_idExpressions.size() != 1) {
        // This transformation is only intended for $group stages that group on a single field.
        return nullptr;
    }

    auto fieldPathExpr = dynamic_cast<ExpressionFieldPath*>(_idExpressions.front().get());
    if (!fieldPathExpr || !fieldPathExpr->isRootFieldPath()) {
        return nullptr;
    }

    const auto fieldPath = fieldPathExpr->getFieldPath();
    if (fieldPath.getPathLength() == 1) {
        // The path is $$CURRENT or $$ROOT. This isn't really a sensible value to group by (since
        // each document has a unique _id, it will just return the entire collection). We only
        // apply the rewrite when grouping by a single field, so we cannot apply it in this case,
        // where we are grouping by the entire document.
        invariant(fieldPath.getFieldName(0) == "CURRENT" || fieldPath.getFieldName(0) == "ROOT");
        return nullptr;
    }

    const auto groupId = fieldPath.tail().fullPath();

    // We can't do this transformation if there are any non-$first accumulators.
    for (auto&& accumulator : _accumulatedFields) {
        if (AccumulatorDocumentsNeeded::kFirstDocument !=
            accumulator.makeAccumulator()->documentsNeeded()) {
            return nullptr;
        }
    }

    std::vector<std::pair<std::string, boost::intrusive_ptr<Expression>>> fields;

    boost::intrusive_ptr<Expression> idField;
    // The _id field can be specified either as a fieldpath (ex. _id: "$a") or as a singleton
    // object (ex. _id: {v: "$a"}).
    if (_idFieldNames.empty()) {
        idField = ExpressionFieldPath::create(pExpCtx.get(), groupId);
    } else {
        invariant(_idFieldNames.size() == 1);
        idField = ExpressionObject::create(pExpCtx.get(),
                                           {{_idFieldNames.front(), _idExpressions.front()}});
    }
    fields.push_back(std::make_pair("_id", idField));

    for (auto&& accumulator : _accumulatedFields) {
        fields.push_back(std::make_pair(accumulator.fieldName, accumulator.expr.argument));

        // Since we don't attempt this transformation for non-$first accumulators,
        // the initializer should always be trivial.
    }

    return GroupFromFirstDocumentTransformation::create(pExpCtx, groupId, std::move(fields));
}


void GroupFromFirstDocumentTransformation::optimize() {
    for (auto&& expr : _accumulatorExprs) {
        expr.second = expr.second->optimize();
    }
}


std::unique_ptr<GroupFromFirstDocumentTransformation> GroupFromFirstDocumentTransformation::create(
    const intrusive_ptr<ExpressionContext>& expCtx,
    const std::string& groupId,
    vector<pair<std::string, intrusive_ptr<Expression>>> accumulatorExprs) {
    return std::make_unique<GroupFromFirstDocumentTransformation>(groupId,
                                                                  std::move(accumulatorExprs));
}