gx_ht_threshold_landscape(byte *contone_align, byte *thresh_align,
                    ht_landscape_info_t *ht_landscape, byte *halftone,
                    int data_length)
{
    __align16 byte contone[LAND_BITS];
    int position_start, position, curr_position;
    int *widths = &(ht_landscape->widths[0]);
    int local_widths[LAND_BITS];
    int num_contone = ht_landscape->num_contones;
    int k, j, w, contone_out_posit;
    byte *contone_ptr, *thresh_ptr, *halftone_ptr;
#ifdef PACIFY_VALGRIND
    int extra = 0;
#endif

    /* Work through chunks of 16.  */
    /* Data may have come in left to right or right to left. */
    if (ht_landscape->index > 0) {
        position = position_start = 0;
    } else {
        position = position_start = ht_landscape->curr_pos + 1;
    }
    thresh_ptr = thresh_align;
    halftone_ptr = halftone;
    /* Copy the widths to a local array, and truncate the last one (which may
     * be the first one!) if required. */
    k = 0;
    for (j = 0; j < num_contone; j++)
        k += (local_widths[j] = widths[position_start+j]);
    if (k > LAND_BITS) {
        if (ht_landscape->index > 0) {
            local_widths[num_contone-1] -= k-LAND_BITS;
        } else {
            local_widths[0] -= k-LAND_BITS;
        }
    }
#ifdef PACIFY_VALGRIND
    if (k < LAND_BITS) {
        extra = LAND_BITS - k;
    }
#endif
    for (k = data_length; k > 0; k--) { /* Loop on rows */
        contone_ptr = &(contone_align[position]); /* Point us to our row start */
        curr_position = 0; /* We use this in keeping track of widths */
        contone_out_posit = 0; /* Our index out */
        for (j = num_contone; j > 0; j--) {
            byte c = *contone_ptr;
            /* The microsoft compiler, cleverly spots that the following loop
             * can be replaced by a memset. Unfortunately, it can't spot that
             * the typical length values of the memset are so small that we'd
             * be better off doing it the slow way. We therefore introduce a
             * sneaky 'volatile' cast below that stops this optimisation. */
            w = local_widths[curr_position];
            do {
                ((volatile byte *)contone)[contone_out_posit] = c;
                contone_out_posit++;
            } while (--w);
#ifdef PACIFY_VALGRIND
            if (extra)
                memset(contone+contone_out_posit, 0, extra);
#endif
            curr_position++; /* Move us to the next position in our width array */
            contone_ptr++;   /* Move us to a new location in our contone buffer */
        }
        /* Now we have our left justified and expanded contone data for
           LAND_BITS/16 sets of 16 bits. Go ahead and threshold these. */
        contone_ptr = &contone[0];
#if LAND_BITS > 16
        j = LAND_BITS;
        do {
#endif
#ifdef HAVE_SSE2
            threshold_16_SSE(contone_ptr, thresh_ptr, halftone_ptr);
#else
            threshold_16_bit(contone_ptr, thresh_ptr, halftone_ptr);
#endif
            thresh_ptr += 16;
            position += 16;
            halftone_ptr += 2;
            contone_ptr += 16;
#if LAND_BITS > 16
            j -= 16;
        } while (j > 0);
#endif
    }
}


Fix bug 697459 Buffer overflow in fill_threshold_buffer

There was an overflow check for ht_buffer size, but none for the larger
threshold_buffer. Note that this file didn't fail on Windows because the
combination of the ht_buffer and the size of the (miscalculated due to
overflow) threshold_buffer would have exceeded the 2Gb limit.

gx_ht_threshold_row_bit(byte *contone,  byte *threshold_strip,  int contone_stride,
                  byte *halftone, int dithered_stride, int width,
                  int num_rows, int offset_bits)
{
#ifndef HAVE_SSE2
    int k, j;
    byte *contone_ptr;
    byte *thresh_ptr;
    byte *halftone_ptr;
    byte bit_init;

    /* For the moment just do a very slow compare until we get
       get this working.  This could use some serious optimization */
    width -= offset_bits;
    for (j = 0; j < num_rows; j++) {
        byte h;
        contone_ptr = contone;
        thresh_ptr = threshold_strip + contone_stride * j;
        halftone_ptr = halftone + dithered_stride * j;
        /* First get the left remainder portion.  Put into MSBs of first byte */
        bit_init = 0x80;
        h = 0;
        k = offset_bits;
        if (k > 0) {
            do {
                if (*contone_ptr++ < *thresh_ptr++) {
                    h |=  bit_init;
                }
                bit_init >>= 1;
                if (bit_init == 0) {
                    bit_init = 0x80;
                    *halftone_ptr++ = h;
                    h = 0;
                }
                k--;
            } while (k > 0);
            bit_init = 0x80;
            *halftone_ptr++ = h;
            h = 0;
            if (offset_bits < 8)
                *halftone_ptr++ = 0;
        }
        /* Now get the rest, which will be 16 bit aligned. */
        k = width;
        if (k > 0) {
            do {
                if (*contone_ptr++ < *thresh_ptr++) {
                    h |=  bit_init;
                }
                bit_init >>= 1;
                if (bit_init == 0) {
                    bit_init = 0x80;
                    *halftone_ptr++ = h;
                    h = 0;
                }
                k--;
            } while (k > 0);
            if (bit_init != 0x80) {
                *halftone_ptr++ = h;
            }
            if ((width & 15) < 8)
                *halftone_ptr++ = 0;
        }
    }
#else
    byte *contone_ptr;
    byte *thresh_ptr;
    byte *halftone_ptr;
    int num_tiles = (width - offset_bits + 15)>>4;
    int k, j;

    for (j = 0; j < num_rows; j++) {
        /* contone and thresh_ptr are 128 bit aligned.  We do need to do this in
           two steps to ensure that we pack the bits in an aligned fashion
           into halftone_ptr.  */
        contone_ptr = contone;
        thresh_ptr = threshold_strip + contone_stride * j;
        halftone_ptr = halftone + dithered_stride * j;
        if (offset_bits > 0) {
            /* Since we allowed for 16 bits in our left remainder
               we can go directly in to the destination.  threshold_16_SSE
               requires 128 bit alignment.  contone_ptr and thresh_ptr
               are set up so that after we move in by offset_bits elements
               then we are 128 bit aligned.  */
            threshold_16_SSE_unaligned(contone_ptr, thresh_ptr,
                                       halftone_ptr);
            halftone_ptr += 2;
            thresh_ptr += offset_bits;
            contone_ptr += offset_bits;
        }
        /* Now we should have 128 bit aligned with our input data. Iterate
           over sets of 16 going directly into our HT buffer.  Sources and
           halftone_ptr buffers should be padded to allow 15 bit overrun */
        for (k = 0; k < num_tiles; k++) {
            threshold_16_SSE(contone_ptr, thresh_ptr, halftone_ptr);
            thresh_ptr += 16;
            contone_ptr += 16;
            halftone_ptr += 2;
        }
    }
#endif
}

The fill_threshhold_buffer function in base/gxht_thresh.c in Artifex Software, Inc. Ghostscript 9.20 allows remote attackers to cause a denial of service (heap-based buffer overflow and application crash) or possibly have unspecified other impact via a crafted PostScript document.


int gxht_dda_length(gx_dda_fixed *dda, int src_size)
{
    gx_dda_fixed d = (*dda);
    dda_advance(d, src_size);
    return abs(fixed2int_var_rounded(dda_current(d)) - fixed2int_var_rounded(dda_current(*dda)));
}


reset_landscape_buffer(ht_landscape_info_t *ht_landscape, byte *contone_align,
                       int data_length, int num_used)
{
    int delta;
    int curr_x_pos = ht_landscape->xstart;

    if (ht_landscape->index < 0) {
        /* Moving right to left, move column to far right */
        delta = ht_landscape->count - num_used;
        memset(&(ht_landscape->widths[0]), 0, sizeof(int)*LAND_BITS);
        ht_landscape->widths[LAND_BITS-1] = delta;
        ht_landscape->curr_pos = LAND_BITS-2;
        ht_landscape->xstart = curr_x_pos - num_used;
    } else {
        /* Moving left to right, move column to far left */
        delta = ht_landscape->count - num_used;
        memset(&(ht_landscape->widths[0]), 0, sizeof(int)*LAND_BITS);
        ht_landscape->widths[0] = delta;
        ht_landscape->curr_pos = 1;
        ht_landscape->xstart = curr_x_pos + num_used;
    }
    ht_landscape->count = delta;
    ht_landscape->num_contones = 1;
}


fill_threshold_buffer(byte *dest_strip, byte *src_strip, int src_width,
                       int left_offset, int left_width, int num_tiles,
                       int right_width)
{
    byte *ptr_out_temp = dest_strip;
    int ii;

    /* Left part */
    memcpy(dest_strip, src_strip + left_offset, left_width);
    ptr_out_temp += left_width;
    /* Now the full parts */
    for (ii = 0; ii < num_tiles; ii++){
        memcpy(ptr_out_temp, src_strip, src_width);
        ptr_out_temp += src_width;
    }
    /* Now the remainder */
    memcpy(ptr_out_temp, src_strip, right_width);
#ifdef PACIFY_VALGRIND
    ptr_out_temp += right_width;
    ii = (dest_strip-ptr_out_temp) % (LAND_BITS-1);
    if (ii > 0)
        memset(ptr_out_temp, 0, ii);
#endif
}


move_landscape_buffer(ht_landscape_info_t *ht_landscape, byte *contone_align,
                       int data_length)
{
    int k;
    int position_curr, position_new;

    if (ht_landscape->index < 0) {
        /* Moving right to left, move column to far right */
        position_curr = ht_landscape->curr_pos + 1;
        position_new = LAND_BITS-1;
    } else {
        /* Moving left to right, move column to far left */
        position_curr = ht_landscape->curr_pos - 1;
        position_new = 0;
    }
    if (position_curr != position_new) {
        for (k = 0; k < data_length; k++) {
                contone_align[position_new] = contone_align[position_curr];
                position_curr += LAND_BITS;
                position_new += LAND_BITS;
        }
    }
}


gxht_thresh_image_init(gx_image_enum *penum)
{
    int code = 0;
    fixed ox;
    int temp;
    int dev_width, max_height;
    int spp_out;
    int k;
    gx_ht_order *d_order;
    gx_dda_fixed dda_ht;

    if (gx_device_must_halftone(penum->dev)) {
        if (penum->pgs != NULL && penum->pgs->dev_ht != NULL) {
            for (k = 0; k < penum->pgs->dev_ht->num_comp; k++) {
                d_order = &(penum->pgs->dev_ht->components[k].corder);
                code = gx_ht_construct_threshold(d_order, penum->dev,
                                                 penum->pgs, k);
                if (code < 0 ) {
                    return gs_rethrow(code, "threshold creation failed");
                }
            }
        } else {
            return -1;
        }
    }
    spp_out = penum->dev->color_info.num_components;
    /* Precompute values needed for rasterizing. */
    penum->dxx = float2fixed(penum->matrix.xx + fixed2float(fixed_epsilon) / 2);
    /* If the image is landscaped then we want to maintain a buffer
       that is sufficiently large so that we can hold a byte
       of halftoned data along the column.  This way we avoid doing
       multiple writes into the same position over and over.
       The size of the buffer we need depends upon the bitdepth of
       the output device, the number of device coloranants and the
       number of  colorants in the source space.  Note we will
       need to eventually  consider  multi-level halftone case
       here too.  For now, to make use of the SSE2 stuff, we would
       like to have a multiple of 16 bytes of data to process at a time.
       So we will collect the columns of data in a buffer that is LAND_BITS
       wide.  We will also keep track of the widths of each column.  When
       the total width count reaches LAND_BITS, we will create our
       threshold array and apply it.  We may have one column that is
       buffered between calls in this case.  Also if a call is made
       with h=0 we will flush the buffer as we are at the end of the
       data.  */
    if (penum->posture == image_landscape) {
        int col_length = fixed2int_var_rounded(any_abs(penum->x_extent.y));
        dda_ht = penum->dda.pixel0.y;
        if (penum->dxx > 0)
            dda_translate(dda_ht, -fixed_epsilon);      /* to match rounding in non-fast code */

        ox = dda_current(penum->dda.pixel0.x);
        temp = gxht_dda_length(&dda_ht, penum->rect.w);
        if (col_length < temp)
            col_length = temp;          /* choose max to make sure line_size is large enough */
        temp = (col_length + LAND_BITS)/LAND_BITS;      /* round up to allow for offset bits */
        /* bitmap_raster() expects the width in bits, hence "* 8" */
        penum->line_size = bitmap_raster((temp * LAND_BITS) * 8);  /* The stride */
        /* Now we need at most LAND_BITS of these */
        penum->line = gs_alloc_bytes(penum->memory,
                                     LAND_BITS * penum->line_size * spp_out + 16,
                                     "gxht_thresh");
        /* Same with this.  However, we only need one plane here */
        penum->thresh_buffer = gs_alloc_bytes(penum->memory,
                                           penum->line_size * LAND_BITS + 16,
                                           "gxht_thresh");
        /* That maps into (LAND_BITS/8) bytes of Halftone data */
        penum->ht_buffer =
                        gs_alloc_bytes(penum->memory,
                           penum->line_size * (LAND_BITS>>3) * spp_out,
                           "gxht_thresh");
        penum->ht_plane_height = penum->line_size;
        penum->ht_stride = penum->line_size;
        if (penum->line == NULL || penum->thresh_buffer == NULL
                    || penum->ht_buffer == NULL)
            return -1;
        penum->ht_landscape.count = 0;
        penum->ht_landscape.num_contones = 0;
        if (penum->y_extent.x < 0) {
            /* Going right to left */
            penum->ht_landscape.curr_pos = LAND_BITS-1;
            penum->ht_landscape.index = -1;
        } else {
            /* Going left to right */
            penum->ht_landscape.curr_pos = 0;
            penum->ht_landscape.index = 1;
        }
        if (penum->x_extent.y < 0) {
            penum->ht_landscape.flipy = true;
            penum->ht_landscape.y_pos =
                fixed2int_pixround_perfect(dda_current(penum->dda.pixel0.y) + penum->x_extent.y);
        } else {
            penum->ht_landscape.flipy = false;
            penum->ht_landscape.y_pos =
                fixed2int_pixround_perfect(dda_current(penum->dda.pixel0.y));
        }
        memset(&(penum->ht_landscape.widths[0]), 0, sizeof(int)*LAND_BITS);
        penum->ht_landscape.offset_set = false;
        penum->ht_offset_bits = 0; /* Will get set in call to render */
        if (code >= 0) {
#if defined(DEBUG) || defined(PACIFY_VALGRIND)
            memset(penum->line, 0, LAND_BITS * penum->line_size * spp_out + 16);
            memset(penum->ht_buffer, 0, penum->line_size * (LAND_BITS>>3) * spp_out);
            memset(penum->thresh_buffer, 0, LAND_BITS * penum->line_size + 16);
#endif
        }
    } else {
        /* In the portrait case we allocate a single line buffer
           in device width, a threshold buffer of the same size
           and possibly wider and the buffer for the halftoned
           bits. We have to do a bit of work to enable 16 byte
           boundary after an offset to ensure that we can make use
           of  the SSE2 operations for thresholding.  We do the
           allocations now to avoid doing them with every line */
        dda_ht = penum->dda.pixel0.x;
        if (penum->dxx > 0)
            dda_translate(dda_ht, -fixed_epsilon);      /* to match rounding in non-fast code */
        /* Initialize the ht_landscape stuff to zero */
        memset(&(penum->ht_landscape), 0, sizeof(ht_landscape_info_t));
        ox = dda_current(dda_ht);
        dev_width = gxht_dda_length(&dda_ht, penum->rect.w);
        /* Get the bit position so that we can do a copy_mono for
           the left remainder and then 16 bit aligned copies for the
           rest.  The right remainder will be OK as it will land in
           the MSBit positions. Note the #define chunk bits16 in
           gdevm1.c.  Allow also for a 15 sample over run.
        */
        penum->ht_offset_bits = (-fixed2int_var_rounded(ox)) & (bitmap_raster(1) - 1);
        if (penum->ht_offset_bits > 0) {
            penum->ht_stride = bitmap_raster((7 + (dev_width + 4)) + (ARCH_SIZEOF_LONG * 8));
        } else {
            penum->ht_stride = bitmap_raster((7 + (dev_width + 2)) + (ARCH_SIZEOF_LONG * 8));
        }
        /* We want to figure out the maximum height that we may
           have in taking a single source row and going to device
           space */
        max_height = (int) ceil(fixed2float(any_abs(penum->dst_height)) /
                                            (float) penum->Height);
        if (max_height <= 0)
            return -1;		/* shouldn't happen, but check so we don't div by zero */
        if (penum->ht_stride * spp_out > max_int / max_height)
            return -1;         /* overflow */

        penum->ht_buffer =
                        gs_alloc_bytes(penum->memory,
                           penum->ht_stride * max_height * spp_out,
                           "gxht_thresh");
        penum->ht_plane_height = penum->ht_stride * max_height;
        /* We want to have 128 bit alignement for our contone and
           threshold strips so that we can use SSE operations
           in the threshold operation.  Add in a minor buffer and offset
           to ensure this.  If gs_alloc_bytes provides at least 16
           bit alignment so we may need to move 14 bytes.  However, the
           HT process is split in two operations.  One that involves
           the HT of a left remainder and the rest which ensures that
           we pack in the HT data in the bits with no skew for a fast
           copy into the gdevm1 device (16 bit copies).  So, we
           need to account for those pixels which occur first and which
           are NOT aligned for the contone buffer.  After we offset
           by this remainder portion we should be 128 bit aligned.
           Also allow a 15 sample over run during the execution.  */
        temp = (int) ceil((float) ((dev_width + 15.0) + 15.0)/16.0);
        penum->line_size = bitmap_raster(temp * 16 * 8);  /* The stride */
        if (penum->line_size > max_int / max_height) {
            gs_free_object(penum->memory, penum->ht_buffer, "gxht_thresh");
            penum->ht_buffer = NULL;
            return -1;         /* thresh_buffer size overflow */
        }
        penum->line = gs_alloc_bytes(penum->memory, penum->line_size * spp_out,
                                     "gxht_thresh");
        penum->thresh_buffer = gs_alloc_bytes(penum->memory,
                                              penum->line_size * max_height,
                                              "gxht_thresh");
        if (penum->line == NULL || penum->thresh_buffer == NULL ||
            penum->ht_buffer == NULL) {
            return -1;
        } else {
#if defined(DEBUG) || defined(PACIFY_VALGRIND)
            memset(penum->line, 0, penum->line_size * spp_out);
            memset(penum->ht_buffer, 0, penum->ht_stride * max_height * spp_out);
            memset(penum->thresh_buffer, 0, penum->line_size * max_height);
#endif
        }
    }
    return code;
}


threshold_16_SSE(byte *contone_ptr, byte *thresh_ptr, byte *ht_data)
{
    __m128i input1;
    __m128i input2;
    register int result_int;
    const unsigned int mask1 = 0x80808080;
    __m128i sign_fix = _mm_set_epi32(mask1, mask1, mask1, mask1);

    /* Load */
    input1 = _mm_load_si128((const __m128i *)contone_ptr);
    input2 = _mm_load_si128((const __m128i *) thresh_ptr);
    /* Unsigned subtraction does Unsigned saturation so we
       have to use the signed operation */
    input1 = _mm_xor_si128(input1, sign_fix);
    input2 = _mm_xor_si128(input2, sign_fix);
    /* Subtract the two */
    input2 = _mm_subs_epi8(input1, input2);
    /* Grab the sign mask */
    result_int = _mm_movemask_epi8(input2);
    /* bit wise reversal on 16 bit word */
    ht_data[0] = bitreverse[(result_int & 0xff)];
    ht_data[1] = bitreverse[((result_int >> 8) & 0xff)];
}


gxht_thresh_planes(gx_image_enum *penum, fixed xrun,
                   int dest_width, int dest_height,
                   byte *thresh_align, gx_device * dev, int offset_contone[],
                   int contone_stride)
{
    int thresh_width, thresh_height, dx;
    int left_rem_end, left_width, vdi;
    int num_full_tiles, right_tile_width;
    int k, jj, dy, j;
    byte *thresh_tile;
    int position;
    bool replicate_tile;
    image_posture posture = penum->posture;
    const int y_pos = penum->yci;
    int width = 0; /* Init to silence compiler warnings */
    byte *ptr_out, *row_ptr, *ptr_out_temp;
    byte *threshold;
    int init_tile, in_row_offset, ii, num_tiles, tile_remainder;
    int offset_bits = penum->ht_offset_bits;
    byte *halftone;
    int dithered_stride = penum->ht_stride;
    bool is_planar_dev = dev->is_planar;
    gx_color_index dev_white = gx_device_white(dev);
    gx_color_index dev_black = gx_device_black(dev);
    int spp_out = dev->color_info.num_components;
    byte *contone_align = NULL; /* Init to silence compiler warnings */

    /* Go ahead and fill the threshold line buffer with tiled threshold values.
       First just grab the row or column that we are going to tile with and
       then do memcpy into the buffer */

    /* Figure out the tile steps.  Left offset, Number of tiles, Right offset. */
    switch (posture) {
        case image_portrait:
            vdi = penum->hci;
            /*  Iterate over the vdi and fill up our threshold buffer.  We
                 also need to loop across the planes of data */
            for (j = 0; j < spp_out; j++) {
                bool threshold_inverted = penum->pgs->dev_ht->components[j].corder.threshold_inverted;

                thresh_width = penum->pgs->dev_ht->components[j].corder.width;
                thresh_height = penum->pgs->dev_ht->components[j].corder.full_height;
                halftone = penum->ht_buffer + j * vdi * dithered_stride;
                /* Compute the tiling positions with dest_width */
                dx = (fixed2int_var_rounded(xrun) + penum->pgs->screen_phase[0].x) % thresh_width;
                /* Left remainder part */
                left_rem_end = min(dx + dest_width, thresh_width);
                /* The left width of our tile part */
                left_width = left_rem_end - dx;
                /* Now the middle part */
                num_full_tiles =
                    (int)fastfloor((dest_width - left_width)/ (float) thresh_width);
                /* Now the right part */
                right_tile_width = dest_width -  num_full_tiles * thresh_width -
                                   left_width;
                /* Get the proper threshold for the colorant count */
                threshold = penum->pgs->dev_ht->components[j].corder.threshold;
                /* Point to the proper contone data */
                contone_align = penum->line + contone_stride * j +
                                offset_contone[j];
                for (k = 0; k < vdi; k++) {
                    /* Get a pointer to our tile row */
                    dy = (penum->yci + k + penum->dev->band_offset_y) % thresh_height;
                    if (dy < 0)
                        dy += thresh_height;
                    thresh_tile = threshold + thresh_width * dy;
                    /* Fill the buffer, can be multiple rows.  Make sure
                       to update with stride */
                    position = contone_stride * k;
                    /* Tile into the 128 bit aligned threshold strip */
                    fill_threshold_buffer(&(thresh_align[position]),
                                           thresh_tile, thresh_width, dx, left_width,
                                           num_full_tiles, right_tile_width);
                }
                /* Apply the threshold operation */
                if (offset_bits > dest_width)
                    offset_bits = dest_width;

                if (threshold_inverted ||
                    (dev->color_info.polarity == GX_CINFO_POLARITY_SUBTRACTIVE && is_planar_dev)) {
                    gx_ht_threshold_row_bit_sub(contone_align, thresh_align, contone_stride,
                                      halftone, dithered_stride, dest_width, vdi,
                                      offset_bits);
                } else {
                    gx_ht_threshold_row_bit(contone_align, thresh_align, contone_stride,
                          halftone, dithered_stride, dest_width, vdi,
                          offset_bits);
                }
            }
            /* FIXME: An improvement here would be to generate the initial
             * offset_bits at the correct offset within the byte so that they
             * align with the remainder of the line. This would mean not
             * always packing them into the first offset_bits (in MSB order)
             * of our 16 bit word, but rather into the last offset_bits
             * (in MSB order) (except when the entire run is small!).
             *
             * This would enable us to do just one aligned copy_mono call for
             * the entire scanline. */
            /* Now do the copy mono or copy plane operation */
            /* First the left remainder bits */
            if (offset_bits > 0) {
                int x_pos = fixed2int_var_rounded(xrun);
                if (!is_planar_dev) {
                    (*dev_proc(dev, copy_mono)) (dev, penum->ht_buffer, 0, dithered_stride,
                                                 gx_no_bitmap_id, x_pos, y_pos,
                                                 offset_bits, vdi, dev_white,
                                                 dev_black);
                } else {
                    (*dev_proc(dev, copy_planes)) (dev, penum->ht_buffer, 0, dithered_stride,
                                                 gx_no_bitmap_id, x_pos, y_pos,
                                                 offset_bits, vdi, vdi);
                }
            }
            if ((dest_width - offset_bits) > 0 ) {
                /* Now the primary aligned bytes */
                int curr_width = dest_width - offset_bits;
                int x_pos = fixed2int_var_rounded(xrun) + offset_bits;
                /* FIXME: This assumes the allowed offset_bits will always be <= 16 */
                int xoffs = offset_bits > 0 ? 16 : 0;

                if (!is_planar_dev) {
                    (*dev_proc(dev, copy_mono)) (dev, penum->ht_buffer, xoffs, dithered_stride,
                                                 gx_no_bitmap_id, x_pos, y_pos,
                                                 curr_width, vdi, dev_white,
                                                 dev_black);
                } else {
                    (*dev_proc(dev, copy_planes)) (dev, penum->ht_buffer, xoffs, dithered_stride,
                                                 gx_no_bitmap_id, x_pos, y_pos,
                                                 curr_width, vdi, vdi);
                }
            }

            break;
        case image_landscape:
            /* Go ahead and paint the chunk if we have LAND_BITS values or a
             * partial to get us in sync with the 1 bit devices 16 bit
             * positions. */
            vdi = penum->wci;
                /* Now do the haftoning into our buffer.  We basically check
                   first if we have enough data or are all done */
            while ( (penum->ht_landscape.count >= LAND_BITS ||
                   ((penum->ht_landscape.count >= offset_bits) &&
                    penum->ht_landscape.offset_set))) {
                /* Go ahead and 2D tile in the threshold buffer at this time */
                /* Always work the tiling from the upper left corner of our
                   LAND_BITS columns */
                for (j = 0; j < spp_out; j++) {
                    halftone = penum->ht_buffer +
                                   j * penum->ht_plane_height * (LAND_BITS>>3);
                    thresh_width = penum->pgs->dev_ht->components[j].corder.width;
                    thresh_height =
                          penum->pgs->dev_ht->components[j].corder.full_height;
                    /* Get the proper threshold for the colorant count */
                    threshold = penum->pgs->dev_ht->components[j].corder.threshold;
                    /* Point to the proper contone data */
                    contone_align = penum->line + offset_contone[j] +
                                      LAND_BITS * j * contone_stride;
                    if (penum->ht_landscape.offset_set) {
                        width = offset_bits;
                    } else {
                        width = LAND_BITS;
                    }
                    if (penum->y_extent.x < 0) {
                        dx = penum->ht_landscape.xstart - width + 1;
                    } else {
                        dx = penum->ht_landscape.xstart;
                    }
                    dx = (dx + penum->pgs->screen_phase[0].x) % thresh_width;
                    dy = (penum->ht_landscape.y_pos +
                              penum->dev->band_offset_y) % thresh_height;
                    if (dy < 0)
                        dy += thresh_height;
                    /* Left remainder part */
                    left_rem_end = min(dx + LAND_BITS, thresh_width);
                    left_width = left_rem_end - dx;
                    /* Now the middle part */
                    num_full_tiles = (LAND_BITS - left_width) / thresh_width;
                    /* Now the right part */
                    right_tile_width =
                        LAND_BITS - num_full_tiles * thresh_width - left_width;
                    /* Now loop over the y stuff */
                    ptr_out = thresh_align;
                    /* Do this in three parts.  We do a top part, followed by
                       larger mem copies followed by a bottom partial. After
                       a slower initial fill we are able to do larger faster
                       expansions */
                    if (dest_height <= 2 * thresh_height) {
                        init_tile = dest_height;
                        replicate_tile = false;
                    } else {
                        init_tile = thresh_height;
                        replicate_tile = true;
                    }
                    for (jj = 0; jj < init_tile; jj++) {
                        in_row_offset = (jj + dy) % thresh_height;
                        row_ptr = threshold + in_row_offset * thresh_width;
                        ptr_out_temp = ptr_out;
                        /* Left part */
                        memcpy(ptr_out_temp, row_ptr + dx, left_width);
                        ptr_out_temp += left_width;
                        /* Now the full tiles */
                        for (ii = 0; ii < num_full_tiles; ii++) {
                            memcpy(ptr_out_temp, row_ptr, thresh_width);
                            ptr_out_temp += thresh_width;
                        }
                        /* Now the remainder */
                        memcpy(ptr_out_temp, row_ptr, right_tile_width);
                        ptr_out += LAND_BITS;
                    }
                    if (replicate_tile) {
                        /* Find out how many we need to copy */
                        num_tiles =
                            (int)fastfloor((float) (dest_height - thresh_height)/ (float) thresh_height);
                        tile_remainder = dest_height - (num_tiles + 1) * thresh_height;
                        for (jj = 0; jj < num_tiles; jj ++) {
                            memcpy(ptr_out, thresh_align, LAND_BITS * thresh_height);
                            ptr_out += LAND_BITS * thresh_height;
                        }
                        /* Now fill in the remainder */
                        memcpy(ptr_out, thresh_align, LAND_BITS * tile_remainder);
                    }
                    /* Apply the threshold operation */
                    if (dev->color_info.polarity == GX_CINFO_POLARITY_SUBTRACTIVE
                        && is_planar_dev) {
                        gx_ht_threshold_landscape_sub(contone_align, thresh_align,
                                            &(penum->ht_landscape), halftone, dest_height);
                    } else {
                        gx_ht_threshold_landscape(contone_align, thresh_align,
                                            &(penum->ht_landscape), halftone, dest_height);
                    }
                    /* We may have a line left over that has to be maintained
                       due to line replication in the resolution conversion. */
                    if (width != penum->ht_landscape.count) {
                        /* move the line do not reset the stuff */
                        move_landscape_buffer(&(penum->ht_landscape),
                                              contone_align, dest_height);
                    }
                }
                /* Perform the copy mono */
                if (penum->ht_landscape.index < 0) {
                    if (!is_planar_dev) {
                        (*dev_proc(dev, copy_mono))
                                       (dev, penum->ht_buffer, 0, LAND_BITS>>3,
                                        gx_no_bitmap_id,
                                        penum->ht_landscape.xstart - width + 1,
                                        penum->ht_landscape.y_pos,
                                        width, dest_height,
                                        dev_white, dev_black);
                    } else {
                        (*dev_proc(dev, copy_planes))
                                       (dev, penum->ht_buffer, 0, LAND_BITS>>3,
                                        gx_no_bitmap_id,
                                        penum->ht_landscape.xstart - width + 1,
                                        penum->ht_landscape.y_pos,
                                        width, dest_height,
                                        penum->ht_plane_height);
                    }
                } else {
                    if (!is_planar_dev) {
                        (*dev_proc(dev, copy_mono)) (dev, penum->ht_buffer, 0, LAND_BITS>>3,
                                                     gx_no_bitmap_id,
                                                     penum->ht_landscape.xstart,
                                                     penum->ht_landscape.y_pos,
                                                     width, dest_height,
                                                     dev_white, dev_black);
                    } else {
                        (*dev_proc(dev, copy_planes)) (dev, penum->ht_buffer, 0, LAND_BITS>>3,
                                                     gx_no_bitmap_id,
                                                     penum->ht_landscape.xstart,
                                                     penum->ht_landscape.y_pos,
                                                     width, dest_height,
                                                     penum->ht_plane_height);
                    }
                }
                penum->ht_landscape.offset_set = false;
                if (width != penum->ht_landscape.count) {
                    reset_landscape_buffer(&(penum->ht_landscape),
                                           contone_align, dest_height,
                                           width);
                } else {
                    /* Reset the whole buffer */
                    penum->ht_landscape.count = 0;
                    if (penum->ht_landscape.index < 0) {
                        /* Going right to left */
                        penum->ht_landscape.curr_pos = LAND_BITS-1;
                    } else {
                        /* Going left to right */
                        penum->ht_landscape.curr_pos = 0;
                    }
                    penum->ht_landscape.num_contones = 0;
                    memset(&(penum->ht_landscape.widths[0]), 0, sizeof(int)*LAND_BITS);
                }
            }
            break;
        default:
            return gs_rethrow(-1, "Invalid orientation for thresholding");
    }
    return 0;
}


gx_ht_threshold_landscape_sub(byte *contone_align, byte *thresh_align,
                    ht_landscape_info_t *ht_landscape, byte *halftone,
                    int data_length)
{
    __align16 byte contone[LAND_BITS];
    int position_start, position, curr_position;
    int *widths = &(ht_landscape->widths[0]);
    int local_widths[LAND_BITS];
    int num_contone = ht_landscape->num_contones;
    int k, j, w, contone_out_posit;
    byte *contone_ptr, *thresh_ptr, *halftone_ptr;
#ifdef PACIFY_VALGRIND
    int extra = 0;
#endif

    /* Work through chunks of 16.  */
    /* Data may have come in left to right or right to left. */
    if (ht_landscape->index > 0) {
        position = position_start = 0;
    } else {
        position = position_start = ht_landscape->curr_pos + 1;
    }
    thresh_ptr = thresh_align;
    halftone_ptr = halftone;
    /* Copy the widths to a local array, and truncate the last one (which may
     * be the first one!) if required. */
    k = 0;
    for (j = 0; j < num_contone; j++)
        k += (local_widths[j] = widths[position_start+j]);
    if (k > LAND_BITS) {
        if (ht_landscape->index > 0) {
            local_widths[num_contone-1] -= k-LAND_BITS;
        } else {
            local_widths[0] -= k-LAND_BITS;
        }
    }
#ifdef PACIFY_VALGRIND
    if (k < LAND_BITS) {
        extra = LAND_BITS - k;
    }
#endif
    for (k = data_length; k > 0; k--) { /* Loop on rows */
        contone_ptr = &(contone_align[position]); /* Point us to our row start */
        curr_position = 0; /* We use this in keeping track of widths */
        contone_out_posit = 0; /* Our index out */
        for (j = num_contone; j > 0; j--) {
            byte c = *contone_ptr;
            /* The microsoft compiler, cleverly spots that the following loop
             * can be replaced by a memset. Unfortunately, it can't spot that
             * the typical length values of the memset are so small that we'd
             * be better off doing it the slow way. We therefore introduce a
             * sneaky 'volatile' cast below that stops this optimisation. */
            w = local_widths[curr_position];
            do {
                ((volatile byte *)contone)[contone_out_posit] = c;
                contone_out_posit++;
            } while (--w);
#ifdef PACIFY_VALGRIND
            if (extra)
                memset(contone+contone_out_posit, 0, extra);
#endif
            curr_position++; /* Move us to the next position in our width array */
            contone_ptr++;   /* Move us to a new location in our contone buffer */
        }
        /* Now we have our left justified and expanded contone data for
           LAND_BITS/16 sets of 16 bits. Go ahead and threshold these. */
        contone_ptr = &contone[0];
#if LAND_BITS > 16
        j = LAND_BITS;
        do {
#endif
#ifdef HAVE_SSE2
            threshold_16_SSE(thresh_ptr, contone_ptr, halftone_ptr);
#else
            threshold_16_bit(thresh_ptr, contone_ptr, halftone_ptr);
#endif
            thresh_ptr += 16;
            position += 16;
            halftone_ptr += 2;
            contone_ptr += 16;
#if LAND_BITS > 16
            j -= 16;
        } while (j > 0);
#endif
    }
}


threshold_16_SSE_unaligned(byte *contone_ptr, byte *thresh_ptr, byte *ht_data)
{
    __m128i input1;
    __m128i input2;
    int result_int;
    byte *sse_data;
    const unsigned int mask1 = 0x80808080;
    __m128i sign_fix = _mm_set_epi32(mask1, mask1, mask1, mask1);

    sse_data = (byte*) &(result_int);
    /* Load */
    input1 = _mm_loadu_si128((const __m128i *)contone_ptr);
    input2 = _mm_loadu_si128((const __m128i *) thresh_ptr);
    /* Unsigned subtraction does Unsigned saturation so we
       have to use the signed operation */
    input1 = _mm_xor_si128(input1, sign_fix);
    input2 = _mm_xor_si128(input2, sign_fix);
    /* Subtract the two */
    input2 = _mm_subs_epi8(input1, input2);
    /* Grab the sign mask */
    result_int = _mm_movemask_epi8(input2);
    /* bit wise reversal on 16 bit word */
    ht_data[0] = bitreverse[sse_data[0]];
    ht_data[1] = bitreverse[sse_data[1]];
}


gx_image_enum_begin(gx_device * dev, const gs_gstate * pgs,
                    const gs_matrix *pmat, const gs_image_common_t * pic,
                const gx_drawing_color * pdcolor, const gx_clip_path * pcpath,
                gs_memory_t * mem, gx_image_enum *penum)
{
    const gs_pixel_image_t *pim = (const gs_pixel_image_t *)pic;
    gs_image_format_t format = pim->format;
    const int width = pim->Width;
    const int height = pim->Height;
    const int bps = pim->BitsPerComponent;
    bool masked = penum->masked;
    const float *decode = pim->Decode;
    gs_matrix_double mat;
    int index_bps;
    const gs_color_space *pcs = pim->ColorSpace;
    gs_logical_operation_t lop = (pgs ? pgs->log_op : lop_default);
    int code;
    int log2_xbytes = (bps <= 8 ? 0 : arch_log2_sizeof_frac);
    int spp, nplanes, spread;
    uint bsize;
    byte *buffer;
    fixed mtx, mty;
    gs_fixed_point row_extent, col_extent, x_extent, y_extent;
    bool device_color = true;
    gs_fixed_rect obox, cbox;
    bool gridfitimages = 0;
    bool in_pattern_accumulator = 0;
    int orthogonal;
    int force_interpolation = 0;

    penum->clues = NULL;
    penum->icc_setup.has_transfer = false;
    penum->icc_setup.is_lab = false;
    penum->icc_setup.must_halftone = false;
    penum->icc_setup.need_decode = false;
    penum->Width = width;
    penum->Height = height;

    if ((code = gx_image_compute_mat(pgs, pmat, &(pim->ImageMatrix), &mat)) < 0) {
        return code;
    }
    /* Grid fit: A common construction in postscript/PDF files is for images
     * to be constructed as a series of 'stacked' 1 pixel high images.
     * Furthermore, many of these are implemented as an imagemask plotted on
     * top of thin rectangles. The different fill rules for images and line
     * art produces problems; line art fills a pixel if any part of it is
     * touched - images only fill a pixel if the centre of the pixel is
     * covered. Bug 692666 is such a problem.
     *
     * As a workaround for this problem, the code below was introduced. The
     * concept is that orthogonal images can be 'grid fitted' (or 'stretch')
     * to entirely cover pixels that they touch. Initially I had this working
     * for all images regardless of type, but as testing has proceeded, this
     * showed more and more regressions, so I've cut the cases back in which
     * this code is used until it now only triggers on imagemasks that are
     * either 1 pixel high, or wide, and then not if we are rendering a
     * glyph (such as from a type3 font).
     */

    /* Ask the device if we are in a pattern accumulator */
    in_pattern_accumulator = (dev_proc(dev, dev_spec_op)(dev, gxdso_in_pattern_accumulator, NULL, 0));
    if (in_pattern_accumulator < 0)
        in_pattern_accumulator = 0;

    /* Figure out if we are orthogonal */
    if (mat.xy == 0 && mat.yx == 0)
        orthogonal = 1;
    else if (mat.xx == 0 && mat.yy == 0)
        orthogonal = 2;
    else
        orthogonal = 0;

    /* If we are in a pattern accumulator, we choose to always grid fit
     * orthogonal images. We do this by asking the device whether we
     * should grid fit. This allows us to avoid nasty blank lines around
     * the edges of cells.
     */
    gridfitimages = in_pattern_accumulator && orthogonal;

    if (pgs != NULL && pgs->show_gstate != NULL) {
        /* If we're a graphics state, and we're in a text object, then we
         * must be in a type3 font. Don't fiddle with it. */
    } else if (!gridfitimages &&
               (!penum->masked || penum->image_parent_type != 0)) {
        /* Other than for images we are specifically looking to grid fit (such as
         * ones in a pattern device), we only grid fit imagemasks */
    } else if (gridfitimages && (penum->masked && penum->image_parent_type == 0)) {
        /* We don't gridfit imagemasks in a pattern accumulator */
    } else if (pgs != NULL && pgs->fill_adjust.x == 0 && pgs->fill_adjust.y == 0) {
        /* If fill adjust is disabled, so is grid fitting */
    } else if (orthogonal == 1) {
        if (width == 1 || gridfitimages) {
            if (mat.xx > 0) {
                fixed ix0 = int2fixed(fixed2int(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.xx * width;
                fixed ix1 = int2fixed(fixed2int_ceiling(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.xx = (double)(fixed2float(ix1 - ix0)/width);
            } else if (mat.xx < 0) {
                fixed ix0 = int2fixed(fixed2int_ceiling(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.xx * width;
                fixed ix1 = int2fixed(fixed2int(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.xx = (double)(fixed2float(ix1 - ix0)/width);
            }
        }
        if (height == 1 || gridfitimages) {
            if (mat.yy > 0) {
                fixed iy0 = int2fixed(fixed2int(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.yy * height;
                fixed iy1 = int2fixed(fixed2int_ceiling(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.yy = (double)(fixed2float(iy1 - iy0)/height);
            } else if (mat.yy < 0) {
                fixed iy0 = int2fixed(fixed2int_ceiling(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.yy * height;
                fixed iy1 = int2fixed(fixed2int(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.yy = ((double)fixed2float(iy1 - iy0)/height);
            }
        }
    } else if (orthogonal == 2) {
        if (height == 1 || gridfitimages) {
            if (mat.yx > 0) {
                fixed ix0 = int2fixed(fixed2int(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.yx * height;
                fixed ix1 = int2fixed(fixed2int_ceiling(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.yx = (double)(fixed2float(ix1 - ix0)/height);
            } else if (mat.yx < 0) {
                fixed ix0 = int2fixed(fixed2int_ceiling(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.yx * height;
                fixed ix1 = int2fixed(fixed2int(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.yx = (double)(fixed2float(ix1 - ix0)/height);
            }
        }
        if (width == 1 || gridfitimages) {
            if (mat.xy > 0) {
                fixed iy0 = int2fixed(fixed2int(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.xy * width;
                fixed iy1 = int2fixed(fixed2int_ceiling(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.xy = (double)(fixed2float(iy1 - iy0)/width);
            } else if (mat.xy < 0) {
                fixed iy0 = int2fixed(fixed2int_ceiling(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.xy * width;
                fixed iy1 = int2fixed(fixed2int(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.xy = ((double)fixed2float(iy1 - iy0)/width);
            }
        }
    }

    /* When rendering to a pattern accumulator, if we are downscaling
     * then enable interpolation, as otherwise dropouts can cause
     * serious problems. */
    if (in_pattern_accumulator) {
        double ome = ((double)(fixed_1 - fixed_epsilon)) / (double)fixed_1; /* One Minus Epsilon */

        if (orthogonal == 1) {
            if ((mat.xx > -ome && mat.xx < ome) || (mat.yy > -ome && mat.yy < ome)) {
                force_interpolation = true;
            }
        } else if (orthogonal == 2) {
            if ((mat.xy > -ome && mat.xy < ome) || (mat.yx > -ome && mat.yx < ome)) {
                force_interpolation = true;
            }
        }
    }

    /* Can we restrict the amount of image we need? */
    while (pcpath) /* So we can break out of it */
    {
        gs_rect rect, rect_out;
        gs_matrix mi;
        const gs_matrix *m = pgs != NULL ? &ctm_only(pgs) : NULL;
        gs_fixed_rect obox;
        gs_int_rect irect;
        if (m == NULL || (code = gs_matrix_invert(m, &mi)) < 0 ||
            (code = gs_matrix_multiply(&mi, &pic->ImageMatrix, &mi)) < 0) {
            /* Give up trying to shrink the render box, but continue processing */
            break;
        }
        gx_cpath_outer_box(pcpath, &obox);
        rect.p.x = fixed2float(obox.p.x);
        rect.p.y = fixed2float(obox.p.y);
        rect.q.x = fixed2float(obox.q.x);
        rect.q.y = fixed2float(obox.q.y);
        code = gs_bbox_transform(&rect, &mi, &rect_out);
        if (code < 0) {
            /* Give up trying to shrink the render box, but continue processing */
            break;
        }
        irect.p.x = (int)(rect_out.p.x-1.0);
        irect.p.y = (int)(rect_out.p.y-1.0);
        irect.q.x = (int)(rect_out.q.x+1.0);
        irect.q.y = (int)(rect_out.q.y+1.0);
        /* Need to expand the region to allow for the fact that the mitchell
         * scaler reads multiple pixels in. This calculation can probably be
         * improved. */
        /* If mi.{xx,yy} > 1 then we are downscaling. During downscaling,
         * the support increases to ensure that we don't lose pixels contributions
         * entirely. */
        /* I do not understand the need for the +/- 1 fudge factors,
         * but they seem to be required. Increasing the render rectangle can
         * never be bad at least... RJW */
        {
            float support = any_abs(mi.xx);
            int isupport;
            if (any_abs(mi.yy) > support)
                support = any_abs(mi.yy);
            if (any_abs(mi.xy) > support)
                support = any_abs(mi.xy);
            if (any_abs(mi.yx) > support)
                support = any_abs(mi.yx);
            isupport = (int)(MAX_ISCALE_SUPPORT * (support+1)) + 1;
            irect.p.x -= isupport;
            irect.p.y -= isupport;
            irect.q.x += isupport;
            irect.q.y += isupport;
        }
        if (penum->rrect.x < irect.p.x) {
            penum->rrect.w -= irect.p.x - penum->rrect.x;
            if (penum->rrect.w < 0)
               penum->rrect.w = 0;
            penum->rrect.x = irect.p.x;
        }
        if (penum->rrect.x + penum->rrect.w > irect.q.x) {
            penum->rrect.w = irect.q.x - penum->rrect.x;
            if (penum->rrect.w < 0)
                penum->rrect.w = 0;
        }
        if (penum->rrect.y < irect.p.y) {
            penum->rrect.h -= irect.p.y - penum->rrect.y;
            if (penum->rrect.h < 0)
                penum->rrect.h = 0;
            penum->rrect.y = irect.p.y;
        }
        if (penum->rrect.y + penum->rrect.h > irect.q.y) {
            penum->rrect.h = irect.q.y - penum->rrect.y;
            if (penum->rrect.h < 0)
                penum->rrect.h = 0;
        }
        break; /* Out of the while */
    }
    /* Check for the intersection being null */
    if (penum->rrect.x + penum->rrect.w <= penum->rect.x  ||
        penum->rect.x  + penum->rect.w  <= penum->rrect.x ||
        penum->rrect.y + penum->rrect.h <= penum->rect.y  ||
        penum->rect.y  + penum->rect.h  <= penum->rrect.y)
    {
          /* Something may have gone wrong with the floating point above.
           * set the region to something sane. */
        penum->rrect.x = penum->rect.x;
        penum->rrect.y = penum->rect.y;
        penum->rrect.w = 0;
        penum->rrect.h = 0;
    }

    /*penum->matrix = mat;*/
    penum->matrix.xx = mat.xx;
    penum->matrix.xy = mat.xy;
    penum->matrix.yx = mat.yx;
    penum->matrix.yy = mat.yy;
    penum->matrix.tx = mat.tx;
    penum->matrix.ty = mat.ty;
    if_debug6m('b', mem, " [%g %g %g %g %g %g]\n",
              mat.xx, mat.xy, mat.yx, mat.yy, mat.tx, mat.ty);
    /* following works for 1, 2, 4, 8, 12, 16 */
    index_bps = (bps < 8 ? bps >> 1 : (bps >> 2) + 1);
    /*
     * Compute extents with distance transformation.
     */
    if (mat.tx > 0)
        mtx = float2fixed(mat.tx);
    else { /* Use positive values to ensure round down. */
        int f = (int)-mat.tx + 1;

        mtx = float2fixed(mat.tx + f) - int2fixed(f);
    }
    if (mat.ty > 0)
        mty = float2fixed(mat.ty);
    else {  /* Use positive values to ensure round down. */
        int f = (int)-mat.ty + 1;

        mty = float2fixed(mat.ty + f) - int2fixed(f);
    }

    row_extent.x = float2fixed_rounded_boxed(width * mat.xx);
    row_extent.y =
        (is_fzero(mat.xy) ? fixed_0 :
         float2fixed_rounded_boxed(width * mat.xy));
    col_extent.x =
        (is_fzero(mat.yx) ? fixed_0 :
         float2fixed_rounded_boxed(height * mat.yx));
    col_extent.y = float2fixed_rounded_boxed(height * mat.yy);
    gx_image_enum_common_init((gx_image_enum_common_t *)penum,
                              (const gs_data_image_t *)pim,
                              &image1_enum_procs, dev,
                              (masked ? 1 : (penum->alpha ? cs_num_components(pcs)+1 : cs_num_components(pcs))),
                              format);
    if (penum->rect.w == width && penum->rect.h == height) {
        x_extent = row_extent;
        y_extent = col_extent;
    } else {
        int rw = penum->rect.w, rh = penum->rect.h;

        x_extent.x = float2fixed_rounded_boxed(rw * mat.xx);
        x_extent.y =
            (is_fzero(mat.xy) ? fixed_0 :
             float2fixed_rounded_boxed(rw * mat.xy));
        y_extent.x =
            (is_fzero(mat.yx) ? fixed_0 :
             float2fixed_rounded_boxed(rh * mat.yx));
        y_extent.y = float2fixed_rounded_boxed(rh * mat.yy);
    }
    /* Set icolor0 and icolor1 to point to image clues locations if we have
       1spp or an imagemask, otherwise image clues is not used and
       we have these values point to other member variables */
    if (masked || cs_num_components(pcs) == 1) {
        /* Go ahead and allocate now if not already done.  For a mask
           we really should only do 2 values. For now, the goal is to
           eliminate the 256 bytes for the >8bpp image enumerator */
        penum->clues = (gx_image_clue*) gs_alloc_bytes(mem, sizeof(gx_image_clue)*256,
                             "gx_image_enum_begin");
        if (penum->clues == NULL)
            return_error(gs_error_VMerror);
        penum->icolor0 = &(penum->clues[0].dev_color);
        penum->icolor1 = &(penum->clues[255].dev_color);
    } else {
        penum->icolor0 = &(penum->icolor0_val);
        penum->icolor1 = &(penum->icolor1_val);
    }
    if (masked) {       /* This is imagemask. */
        if (bps != 1 || pcs != NULL || penum->alpha || decode[0] == decode[1]) {
            return_error(gs_error_rangecheck);
        }
        /* Initialize color entries 0 and 255. */
        set_nonclient_dev_color(penum->icolor0, gx_no_color_index);
        set_nonclient_dev_color(penum->icolor1, gx_no_color_index);
        *(penum->icolor1) = *pdcolor;
        memcpy(&penum->map[0].table.lookup4x1to32[0],
               (decode[0] < decode[1] ? lookup4x1to32_inverted :
                lookup4x1to32_identity),
               16 * 4);
        penum->map[0].decoding = sd_none;
        spp = 1;
        lop = rop3_know_S_0(lop);
    } else {                    /* This is image, not imagemask. */
        const gs_color_space_type *pcst = pcs->type;
        int b_w_color;

        spp = cs_num_components(pcs);
        if (spp < 0) {          /* Pattern not allowed */
            return_error(gs_error_rangecheck);
        }
        if (penum->alpha)
            ++spp;
        /* Use a less expensive format if possible. */
        switch (format) {
        case gs_image_format_bit_planar:
            if (bps > 1)
                break;
            format = gs_image_format_component_planar;
        case gs_image_format_component_planar:
            if (spp == 1)
                format = gs_image_format_chunky;
        default:                /* chunky */
            break;
        }

        if (pcs->cmm_icc_profile_data != NULL) {
            device_color = false;
        } else {
            device_color = (*pcst->concrete_space) (pcs, pgs) == pcs;
        }

        code = image_init_colors(penum, bps, spp, format, decode, pgs, dev,
                          pcs, &device_color);
        if (code < 0)
            return gs_throw(code, "Image colors initialization failed");
        /* If we have a CIE based color space and the icc equivalent profile
           is not yet set, go ahead and handle that now.  It may already
           be done due to the above init_colors which may go through remap. */
        if (gs_color_space_is_PSCIE(pcs) && pcs->icc_equivalent == NULL) {
            code = gs_colorspace_set_icc_equivalent((gs_color_space *)pcs, &(penum->icc_setup.is_lab),
                                                pgs->memory);
            if (code < 0)
                return code;
            if (penum->icc_setup.is_lab) {
                /* Free what ever profile was created and use the icc manager's
                   cielab profile */
                gs_color_space *curr_pcs = (gs_color_space *)pcs;
                rc_decrement(curr_pcs->icc_equivalent,"gx_image_enum_begin");
                rc_decrement(curr_pcs->cmm_icc_profile_data,"gx_image_enum_begin");
                curr_pcs->cmm_icc_profile_data = pgs->icc_manager->lab_profile;
                rc_increment(curr_pcs->cmm_icc_profile_data);
            }
        }
        /* Try to transform non-default RasterOps to something */
        /* that we implement less expensively. */
        if (!pim->CombineWithColor)
            lop = rop3_know_T_0(lop) & ~lop_T_transparent;
        else if ((rop3_uses_T(lop) && color_draws_b_w(dev, pdcolor) == 0))
            lop = rop3_know_T_0(lop);

        if (lop != rop3_S &&    /* if best case, no more work needed */
            !rop3_uses_T(lop) && bps == 1 && spp == 1 &&
            (b_w_color =
             color_draws_b_w(dev, penum->icolor0)) >= 0 &&
            color_draws_b_w(dev, penum->icolor1) == (b_w_color ^ 1)
            ) {
            if (b_w_color) {    /* Swap the colors and invert the RasterOp source. */
                gx_device_color dcolor;

                dcolor = *(penum->icolor0);
                *(penum->icolor0) = *(penum->icolor1);
                *(penum->icolor1) = dcolor;
                lop = rop3_invert_S(lop);
            }
            /*
             * At this point, we know that the source pixels
             * correspond directly to the S input for the raster op,
             * i.e., icolor0 is black and icolor1 is white.
             */
            switch (lop) {
                case rop3_D & rop3_S:
                    /* Implement this as an inverted mask writing 0s. */
                    *(penum->icolor1) = *(penum->icolor0);
                    /* (falls through) */
                case rop3_D | rop3_not(rop3_S):
                    /* Implement this as an inverted mask writing 1s. */
                    memcpy(&penum->map[0].table.lookup4x1to32[0],
                           lookup4x1to32_inverted, 16 * 4);
                  rmask:        /* Fill in the remaining parameters for a mask. */
                    penum->masked = masked = true;
                    set_nonclient_dev_color(penum->icolor0, gx_no_color_index);
                    penum->map[0].decoding = sd_none;
                    lop = rop3_T;
                    break;
                case rop3_D & rop3_not(rop3_S):
                    /* Implement this as a mask writing 0s. */
                    *(penum->icolor1) = *(penum->icolor0);
                    /* (falls through) */
                case rop3_D | rop3_S:
                    /* Implement this as a mask writing 1s. */
                    memcpy(&penum->map[0].table.lookup4x1to32[0],
                           lookup4x1to32_identity, 16 * 4);
                    goto rmask;
                default:
                    ;
            }
        }
    }
    penum->device_color = device_color;
    /*
     * Adjust width upward for unpacking up to 7 trailing bits in
     * the row, plus 1 byte for end-of-run, plus up to 7 leading
     * bits for data_x offset within a packed byte.
     */
    bsize = ((bps > 8 ? width * 2 : width) + 15) * spp;
    buffer = gs_alloc_bytes(mem, bsize, "image buffer");
    if (buffer == 0) {
        return_error(gs_error_VMerror);
    }
    penum->bps = bps;
    penum->unpack_bps = bps;
    penum->log2_xbytes = log2_xbytes;
    penum->spp = spp;
    switch (format) {
    case gs_image_format_chunky:
        nplanes = 1;
        spread = 1 << log2_xbytes;
        break;
    case gs_image_format_component_planar:
        nplanes = spp;
        spread = spp << log2_xbytes;
        break;
    case gs_image_format_bit_planar:
        nplanes = spp * bps;
        spread = spp << log2_xbytes;
        break;
    default:
        /* No other cases are possible (checked by gx_image_enum_alloc). */
        return_error(gs_error_Fatal);
    }
    penum->num_planes = nplanes;
    penum->spread = spread;
    /*
     * If we're asked to interpolate in a partial image, we have to
     * assume that the client either really only is interested in
     * the given sub-image, or else is constructing output out of
     * overlapping pieces.
     */
    penum->interpolate = force_interpolation ? interp_force : pim->Interpolate ? interp_on : interp_off;
    penum->x_extent = x_extent;
    penum->y_extent = y_extent;
    penum->posture =
        ((x_extent.y | y_extent.x) == 0 ? image_portrait :
         (x_extent.x | y_extent.y) == 0 ? image_landscape :
         image_skewed);
    penum->pgs = pgs;
    penum->pcs = pcs;
    penum->memory = mem;
    penum->buffer = buffer;
    penum->buffer_size = bsize;
    penum->line = NULL;
    penum->icc_link = NULL;
    penum->color_cache = NULL;
    penum->ht_buffer = NULL;
    penum->thresh_buffer = NULL;
    penum->use_cie_range = false;
    penum->line_size = 0;
    penum->use_rop = lop != (masked ? rop3_T : rop3_S);
#ifdef DEBUG
    if (gs_debug_c('*')) {
        if (penum->use_rop)
            dmprintf1(mem, "[%03x]", lop);
        dmprintf5(mem, "%c%d%c%dx%d ",
                 (masked ? (color_is_pure(pdcolor) ? 'm' : 'h') : 'i'),
                 bps,
                 (penum->posture == image_portrait ? ' ' :
                  penum->posture == image_landscape ? 'L' : 'T'),
                 width, height);
    }
#endif
    penum->slow_loop = 0;
    if (pcpath == 0) {
        (*dev_proc(dev, get_clipping_box)) (dev, &obox);
        cbox = obox;
        penum->clip_image = 0;
    } else
        penum->clip_image =
            (gx_cpath_outer_box(pcpath, &obox) |        /* not || */
             gx_cpath_inner_box(pcpath, &cbox) ?
             0 : image_clip_region);
    penum->clip_outer = obox;
    penum->clip_inner = cbox;
    penum->log_op = rop3_T;     /* rop device takes care of this */
    penum->clip_dev = 0;        /* in case we bail out */
    penum->rop_dev = 0;         /* ditto */
    penum->scaler = 0;          /* ditto */
    /*
     * If all four extrema of the image fall within the clipping
     * rectangle, clipping is never required.  When making this check,
     * we must carefully take into account the fact that we only care
     * about pixel centers.
     */
    {
        fixed
            epx = min(row_extent.x, 0) + min(col_extent.x, 0),
            eqx = max(row_extent.x, 0) + max(col_extent.x, 0),
            epy = min(row_extent.y, 0) + min(col_extent.y, 0),
            eqy = max(row_extent.y, 0) + max(col_extent.y, 0);

        {
            int hwx, hwy;

            switch (penum->posture) {
                case image_portrait:
                    hwx = width, hwy = height;
                    break;
                case image_landscape:
                    hwx = height, hwy = width;
                    break;
                default:
                    hwx = hwy = 0;
            }
            /*
             * If the image is only 1 sample wide or high,
             * and is less than 1 device pixel wide or high,
             * move it slightly so that it covers pixel centers.
             * This is a hack to work around a bug in some old
             * versions of TeX/dvips, which use 1-bit-high images
             * to draw horizontal and vertical lines without
             * positioning them properly.
             */
            if (hwx == 1 && eqx - epx < fixed_1) {
                fixed diff =
                arith_rshift_1(row_extent.x + col_extent.x);

                mtx = (((mtx + diff) | fixed_half) & -fixed_half) - diff;
            }
            if (hwy == 1 && eqy - epy < fixed_1) {
                fixed diff =
                arith_rshift_1(row_extent.y + col_extent.y);

                mty = (((mty + diff) | fixed_half) & -fixed_half) - diff;
            }
        }
        if_debug5m('b', mem, "[b]Image: %sspp=%d, bps=%d, mt=(%g,%g)\n",
                   (masked? "masked, " : ""), spp, bps,
                   fixed2float(mtx), fixed2float(mty));
        if_debug9m('b', mem,
                   "[b]   cbox=(%g,%g),(%g,%g), obox=(%g,%g),(%g,%g), clip_image=0x%x\n",
                   fixed2float(cbox.p.x), fixed2float(cbox.p.y),
                   fixed2float(cbox.q.x), fixed2float(cbox.q.y),
                   fixed2float(obox.p.x), fixed2float(obox.p.y),
                   fixed2float(obox.q.x), fixed2float(obox.q.y),
                   penum->clip_image);
        /* These DDAs enumerate the starting position of each source pixel
         * row in device space. */
        dda_init(penum->dda.row.x, mtx, col_extent.x, height);
        dda_init(penum->dda.row.y, mty, col_extent.y, height);
        if (penum->posture == image_portrait) {
            penum->dst_width = row_extent.x;
            penum->dst_height = col_extent.y;
        } else {
            penum->dst_width = col_extent.x;
            penum->dst_height = row_extent.y;
        }
        /* For gs_image_class_0_interpolate. */
        penum->yi0 = fixed2int_pixround_perfect(dda_current(penum->dda.row.y)); /* For gs_image_class_0_interpolate. */
        if (penum->rect.y) {
            int y = penum->rect.y;

            while (y--) {
                dda_next(penum->dda.row.x);
                dda_next(penum->dda.row.y);
            }
        }
        penum->cur.x = penum->prev.x = dda_current(penum->dda.row.x);
        penum->cur.y = penum->prev.y = dda_current(penum->dda.row.y);
        /* These DDAs enumerate the starting positions of each row of our
         * source pixel data, in the subrectangle ('strip') that we are
         * actually rendering. */
        dda_init(penum->dda.strip.x, penum->cur.x, row_extent.x, width);
        dda_init(penum->dda.strip.y, penum->cur.y, row_extent.y, width);
        if (penum->rect.x) {
            dda_advance(penum->dda.strip.x, penum->rect.x);
            dda_advance(penum->dda.strip.y, penum->rect.x);
        }
        {
            fixed ox = dda_current(penum->dda.strip.x);
            fixed oy = dda_current(penum->dda.strip.y);

            if (!penum->clip_image)     /* i.e., not clip region */
                penum->clip_image =
                    (fixed_pixround(ox + epx) < fixed_pixround(cbox.p.x) ?
                     image_clip_xmin : 0) +
                    (fixed_pixround(ox + eqx) >= fixed_pixround(cbox.q.x) ?
                     image_clip_xmax : 0) +
                    (fixed_pixround(oy + epy) < fixed_pixround(cbox.p.y) ?
                     image_clip_ymin : 0) +
                    (fixed_pixround(oy + eqy) >= fixed_pixround(cbox.q.y) ?
                     image_clip_ymax : 0);
        }
    }
    penum->y = 0;
    penum->used.x = 0;
    penum->used.y = 0;
    {
        static sample_unpack_proc_t procs[2][6] = {
        {   sample_unpack_1, sample_unpack_2,
            sample_unpack_4, sample_unpack_8,
            sample_unpack_12, sample_unpack_16
        },
        {   sample_unpack_1_interleaved, sample_unpack_2_interleaved,
            sample_unpack_4_interleaved, sample_unpack_8_interleaved,
            sample_unpack_12, sample_unpack_16
        }};
        int num_planes = penum->num_planes;
        bool interleaved = (num_planes == 1 && penum->plane_depths[0] != penum->bps);
        int i;

        if (interleaved) {
            int num_components = penum->plane_depths[0] / penum->bps;

            for (i = 1; i < num_components; i++) {
                if (decode[0] != decode[i * 2 + 0] ||
                    decode[1] != decode[i * 2 + 1])
                    break;
            }
            if (i == num_components)
                interleaved = false; /* Use single table. */
        }
        penum->unpack = procs[interleaved][index_bps];

        if_debug1m('b', mem, "[b]unpack=%d\n", bps);
        /* Set up pixel0 for image class procedures. */
        penum->dda.pixel0 = penum->dda.strip;
        for (i = 0; i < gx_image_class_table_count; ++i)
            if ((penum->render = gx_image_class_table[i](penum)) != 0)
                break;
        if (i == gx_image_class_table_count) {
            /* No available class can handle this image. */
            gx_default_end_image(dev, (gx_image_enum_common_t *) penum,
                                 false);
            return_error(gs_error_rangecheck);
        }
    }
    if (penum->clip_image && pcpath) {  /* Set up the clipping device. */
        gx_device_clip *cdev =
            gs_alloc_struct(mem, gx_device_clip,
                            &st_device_clip, "image clipper");

        if (cdev == 0) {
            gx_default_end_image(dev,
                                 (gx_image_enum_common_t *) penum,
                                 false);
            return_error(gs_error_VMerror);
        }
        gx_make_clip_device_in_heap(cdev, pcpath, dev, mem);
        penum->clip_dev = cdev;
    }
    if (penum->use_rop) {       /* Set up the RasterOp source device. */
        gx_device_rop_texture *rtdev;

        code = gx_alloc_rop_texture_device(&rtdev, mem,
                                           "image RasterOp");
        if (code < 0) {
            gx_default_end_image(dev, (gx_image_enum_common_t *) penum,
                                 false);
            return code;
        }
        /* The 'target' must not be NULL for gx_make_rop_texture_device */
        if (!penum->clip_dev && !dev)
            return_error(gs_error_undefined);

        gx_make_rop_texture_device(rtdev,
                                   (penum->clip_dev != 0 ?
                                    (gx_device *) penum->clip_dev :
                                    dev), lop, pdcolor);
        gx_device_retain((gx_device *)rtdev, true);
        penum->rop_dev = rtdev;
    }
    return 0;
}


threshold_16_bit(byte *contone_ptr, byte *thresh_ptr, byte *ht_data)
{
    int j;

    for (j = 2; j > 0; j--) {
        byte h = 0;
        byte bit_init = 0x80;
        do {
            if (*contone_ptr++ < *thresh_ptr++) {
                h |=  bit_init;
            }
            bit_init >>= 1;
        } while (bit_init != 0);
        *ht_data++ = h;
    }
}


Bug 701815: avoid uninitialised bytes being >7, which broke indexing.

Fixes:
    ./sanbin/gs -dBATCH -dNOPAUSE -sOutputFile=tmp -sDEVICE=jetp3852 ../bug-701815.pdf

jetp3852_print_page(gx_device_printer *pdev, gp_file *prn_stream)
{
#define DATA_SIZE (LINE_SIZE * 8)

    unsigned int cnt_2prn;
    unsigned int count,tempcnt;
    unsigned char vtp,cntc1,cntc2;
    int line_size_color_plane;

    byte data[DATA_SIZE];
    byte plane_data[LINE_SIZE * 3];

    /* Initialise data to zeros, otherwise later on, uninitialised bytes in
    dp[] can be greater than 7, which breaks spr8[dp[]]. */
    memset(data, 0x00, DATA_SIZE);

    /* Set initial condition for printer */
    gp_fputs("\033@",prn_stream);

    /* Send each scan line in turn */
    {
        int lnum;
        int line_size = gdev_mem_bytes_per_scan_line((gx_device *)pdev);
        int num_blank_lines = 0;

        if (line_size > DATA_SIZE) {
            emprintf2(pdev->memory, "invalid resolution and/or width gives line_size = %d, max. is %d\n",
                      line_size, DATA_SIZE);
            return_error(gs_error_rangecheck);
        }

        for ( lnum = 0; lnum < pdev->height; lnum++ ) {
            byte *end_data = data + line_size;
            gdev_prn_copy_scan_lines(pdev, lnum,
                                     (byte *)data, line_size);
            /* Remove trailing 0s. */
            while ( end_data > data && end_data[-1] == 0 )
                end_data--;
            if ( end_data == data ) {
                /* Blank line */
                num_blank_lines++;
            } else {
                int i;
                byte *odp;
                byte *row;

                /* Transpose the data to get pixel planes. */
                for ( i = 0, odp = plane_data; i < DATA_SIZE;
                      i += 8, odp++
                    ) { /* The following is for 16-bit machines */
#define spread3(c)\
 { 0, c, c*0x100, c*0x101, c*0x10000L, c*0x10001L, c*0x10100L, c*0x10101L }
                    static ulong spr40[8] = spread3(0x40);
                    static ulong spr8[8] = spread3(8);
                    static ulong spr2[8] = spread3(2);
                    register byte *dp = data + i;
                    register ulong pword =
                                     (spr40[dp[0]] << 1) +
                                     (spr40[dp[1]]) +
                                     (spr40[dp[2]] >> 1) +
                                     (spr8[dp[3]] << 1) +
                                     (spr8[dp[4]]) +
                                     (spr8[dp[5]] >> 1) +
                                     (spr2[dp[6]]) +
                                     (spr2[dp[7]] >> 1);
                    odp[0] = (byte)(pword >> 16);
                    odp[LINE_SIZE] = (byte)(pword >> 8);
                    odp[LINE_SIZE*2] = (byte)(pword);
                }
                /* Skip blank lines if any */
                if ( num_blank_lines > 0 ) {
                    /* Do "dot skips" */
                    while(num_blank_lines > 255) {
                        gp_fputs("\033e\377",prn_stream);
                        num_blank_lines -= 255;
                    }
                    vtp = num_blank_lines;
                    gp_fprintf(prn_stream,"\033e%c",vtp);
                    num_blank_lines = 0;
                }

                /* Transfer raster graphics in the order R, G, B. */
                /* Apparently it is stored in B, G, R */
                /* Calculate the amount of data to send by what */
                /* Ghostscript tells us the scan line_size in (bytes) */

                count = line_size / 3;
                line_size_color_plane = count / 3;
                cnt_2prn = line_size_color_plane * 3 + 5;
                tempcnt = cnt_2prn;
                cntc1 = (tempcnt & 0xFF00) >> 8;
                cntc2 = (tempcnt & 0x00FF);
                gp_fprintf(prn_stream, "\033[O%c%c\200\037",cntc2,cntc1);
                gp_fputc('\000',prn_stream);
                gp_fputs("\124\124",prn_stream);

                for ( row = plane_data + LINE_SIZE * 2, i = 0;
                      i < 3; row -= LINE_SIZE, i++ ) {
                    int jj;
                    byte ctemp;
                    odp = row;
                    /* Complement bytes */
                    for (jj=0; jj< line_size_color_plane; jj++) {
                        ctemp = *odp;
                        *odp++ = ~ctemp;
                    }
                    gp_fwrite(row, sizeof(byte),
                              line_size_color_plane, prn_stream);
                }
            }
        }
    }

    /* eject page */
    gp_fputs("\014", prn_stream);

    return 0;
}

A buffer overflow vulnerability in jetp3852_print_page() in devices/gdev3852.c of Artifex Software GhostScript v9.50 allows a remote attacker to escalate privileges via a crafted PDF file. This is fixed in v9.51.


io_uring: reexpand under-reexpanded iters

[   74.211232] BUG: KASAN: stack-out-of-bounds in iov_iter_revert+0x809/0x900
[   74.212778] Read of size 8 at addr ffff888025dc78b8 by task
syz-executor.0/828
[   74.214756] CPU: 0 PID: 828 Comm: syz-executor.0 Not tainted
5.14.0-rc3-next-20210730 #1
[   74.216525] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996),
BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014
[   74.219033] Call Trace:
[   74.219683]  dump_stack_lvl+0x8b/0xb3
[   74.220706]  print_address_description.constprop.0+0x1f/0x140
[   74.224226]  kasan_report.cold+0x7f/0x11b
[   74.226085]  iov_iter_revert+0x809/0x900
[   74.227960]  io_write+0x57d/0xe40
[   74.232647]  io_issue_sqe+0x4da/0x6a80
[   74.242578]  __io_queue_sqe+0x1ac/0xe60
[   74.245358]  io_submit_sqes+0x3f6e/0x76a0
[   74.248207]  __do_sys_io_uring_enter+0x90c/0x1a20
[   74.257167]  do_syscall_64+0x3b/0x90
[   74.257984]  entry_SYSCALL_64_after_hwframe+0x44/0xae

old_size = iov_iter_count();
...
iov_iter_revert(old_size - iov_iter_count());

If iov_iter_revert() is done base on the initial size as above, and the
iter is truncated and not reexpanded in the middle, it miscalculates
borders causing problems. This trace is due to no one reexpanding after
generic_write_checks().

Now iters store how many bytes has been truncated, so reexpand them to
the initial state right before reverting.

Cc: stable@vger.kernel.org
Reported-by: Palash Oswal <oswalpalash@gmail.com>
Reported-by: Sudip Mukherjee <sudipm.mukherjee@gmail.com>
Reported-and-tested-by: syzbot+9671693590ef5aad8953@syzkaller.appspotmail.com
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
{
	__s32 __user *fds = arg;
	int fd;

	if (ctx->cq_ev_fd)
		return -EBUSY;

	if (copy_from_user(&fd, fds, sizeof(*fds)))
		return -EFAULT;

	ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
	if (IS_ERR(ctx->cq_ev_fd)) {
		int ret = PTR_ERR(ctx->cq_ev_fd);

		ctx->cq_ev_fd = NULL;
		return ret;
	}

	return 0;

An out-of-bounds read flaw was found in the Linux kernel’s io_uring module in the way a user triggers the io_read() function with some special parameters. This flaw allows a local user to read some memory out of bounds.


static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_provide_buf *p = &req->pbuf;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer *head;
	int ret = 0;
	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;

	io_ring_submit_lock(ctx, !force_nonblock);

	lockdep_assert_held(&ctx->uring_lock);

	ret = -ENOENT;
	head = xa_load(&ctx->io_buffers, p->bgid);
	if (head)
		ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
	if (ret < 0)
		req_set_fail(req);

	/* complete before unlock, IOPOLL may need the lock */
	__io_req_complete(req, issue_flags, ret, 0);
	io_ring_submit_unlock(ctx, !force_nonblock);
	return 0;
}


static void io_req_task_cancel(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	/* ctx is guaranteed to stay alive while we hold uring_lock */
	mutex_lock(&ctx->uring_lock);
	io_req_complete_failed(req, req->result);
	mutex_unlock(&ctx->uring_lock);
}


static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
{
	struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
	struct io_ring_ctx *ctx = rsrc_data->ctx;
	struct io_rsrc_put *prsrc, *tmp;

	list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
		list_del(&prsrc->list);

		if (prsrc->tag) {
			bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;

			io_ring_submit_lock(ctx, lock_ring);
			spin_lock_irq(&ctx->completion_lock);
			io_cqring_fill_event(ctx, prsrc->tag, 0, 0);
			ctx->cq_extra++;
			io_commit_cqring(ctx);
			spin_unlock_irq(&ctx->completion_lock);
			io_cqring_ev_posted(ctx);
			io_ring_submit_unlock(ctx, lock_ring);
		}

		rsrc_data->do_put(ctx, prsrc);
		kfree(prsrc);
	}

	io_rsrc_node_destroy(ref_node);
	if (atomic_dec_and_test(&rsrc_data->refs))
		complete(&rsrc_data->done);


static void io_iopoll_req_issued(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
	const bool in_async = io_wq_current_is_worker();

	/* workqueue context doesn't hold uring_lock, grab it now */
	if (unlikely(in_async))
		mutex_lock(&ctx->uring_lock);

	/*
	 * Track whether we have multiple files in our lists. This will impact
	 * how we do polling eventually, not spinning if we're on potentially
	 * different devices.
	 */
	if (list_empty(&ctx->iopoll_list)) {
		ctx->poll_multi_queue = false;
	} else if (!ctx->poll_multi_queue) {
		struct io_kiocb *list_req;
		unsigned int queue_num0, queue_num1;

		list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
						inflight_entry);

		if (list_req->file != req->file) {
			ctx->poll_multi_queue = true;
		} else {
			queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie);
			queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie);
			if (queue_num0 != queue_num1)
				ctx->poll_multi_queue = true;
		}
	}

	/*
	 * For fast devices, IO may have already completed. If it has, add
	 * it to the front so we find it first.
	 */
	if (READ_ONCE(req->iopoll_completed))
		list_add(&req->inflight_entry, &ctx->iopoll_list);
	else
		list_add_tail(&req->inflight_entry, &ctx->iopoll_list);

	if (unlikely(in_async)) {
		/*
		 * If IORING_SETUP_SQPOLL is enabled, sqes are either handle
		 * in sq thread task context or in io worker task context. If
		 * current task context is sq thread, we don't need to check
		 * whether should wake up sq thread.
		 */
		if ((ctx->flags & IORING_SETUP_SQPOLL) &&
		    wq_has_sleeper(&ctx->sq_data->wait))
			wake_up(&ctx->sq_data->wait);

		mutex_unlock(&ctx->uring_lock);
	}
}


static void io_poll_task_func(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_kiocb *nxt;

	if (io_poll_rewait(req, &req->poll)) {
		spin_unlock_irq(&ctx->completion_lock);
	} else {
		bool done;

		done = io_poll_complete(req, req->result);
		if (done) {
			hash_del(&req->hash_node);
		} else {
			req->result = 0;
			add_wait_queue(req->poll.head, &req->poll.wait);
		}
		spin_unlock_irq(&ctx->completion_lock);
		io_cqring_ev_posted(ctx);

		if (done) {
			nxt = io_put_req_find_next(req);
			if (nxt)
				io_req_task_submit(nxt);
		}
	}


static inline void req_set_fail(struct io_kiocb *req)
{
	req->flags |= REQ_F_FAIL;
}


static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_async_msghdr iomsg, *kmsg;
	struct socket *sock;
	unsigned flags;
	int min_ret = 0;
	int ret;

	sock = sock_from_file(req->file);
	if (unlikely(!sock))
		return -ENOTSOCK;

	kmsg = req->async_data;
	if (!kmsg) {
		ret = io_sendmsg_copy_hdr(req, &iomsg);
		if (ret)
			return ret;
		kmsg = &iomsg;
	}

	flags = req->sr_msg.msg_flags;
	if (issue_flags & IO_URING_F_NONBLOCK)
		flags |= MSG_DONTWAIT;
	if (flags & MSG_WAITALL)
		min_ret = iov_iter_count(&kmsg->msg.msg_iter);

	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
	if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
		return io_setup_async_msg(req, kmsg);
	if (ret == -ERESTARTSYS)
		ret = -EINTR;

	/* fast path, check for non-NULL to avoid function call */
	if (kmsg->free_iov)
		kfree(kmsg->free_iov);
	req->flags &= ~REQ_F_NEED_CLEANUP;
	if (ret < min_ret)
		req_set_fail(req);
	__io_req_complete(req, issue_flags, ret, 0);
	return 0;
}


static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
{
	if (req->file->f_op->read_iter)
		return call_read_iter(req->file, &req->rw.kiocb, iter);
	else if (req->file->f_op->read)
		return loop_rw_iter(READ, req, iter);
	else
		return -EINVAL;
}


static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
{
	unsigned int to_submit;
	int ret = 0;

	to_submit = io_sqring_entries(ctx);
	/* if we're handling multiple rings, cap submit size for fairness */
	if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE)
		to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE;

	if (!list_empty(&ctx->iopoll_list) || to_submit) {
		unsigned nr_events = 0;
		const struct cred *creds = NULL;

		if (ctx->sq_creds != current_cred())
			creds = override_creds(ctx->sq_creds);

		mutex_lock(&ctx->uring_lock);
		if (!list_empty(&ctx->iopoll_list))
			io_do_iopoll(ctx, &nr_events, 0);

		/*
		 * Don't submit if refs are dying, good for io_uring_register(),
		 * but also it is relied upon by io_ring_exit_work()
		 */
		if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
		    !(ctx->flags & IORING_SETUP_R_DISABLED))
			ret = io_submit_sqes(ctx, to_submit);
		mutex_unlock(&ctx->uring_lock);

		if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
			wake_up(&ctx->sqo_sq_wait);
		if (creds)
			revert_creds(creds);
	}

	return ret;


static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
{
	unsigned long flags;
	bool all_flushed, posted;

	if (!force && __io_cqring_events(ctx) == ctx->cq_entries)
		return false;

	posted = false;
	spin_lock_irqsave(&ctx->completion_lock, flags);
	while (!list_empty(&ctx->cq_overflow_list)) {
		struct io_uring_cqe *cqe = io_get_cqe(ctx);
		struct io_overflow_cqe *ocqe;

		if (!cqe && !force)
			break;
		ocqe = list_first_entry(&ctx->cq_overflow_list,
					struct io_overflow_cqe, list);
		if (cqe)
			memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
		else
			io_account_cq_overflow(ctx);

		posted = true;
		list_del(&ocqe->list);
		kfree(ocqe);
	}

	all_flushed = list_empty(&ctx->cq_overflow_list);
	if (all_flushed) {
		clear_bit(0, &ctx->check_cq_overflow);
		ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
	}

	if (posted)
		io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	if (posted)
		io_cqring_ev_posted(ctx);
	return all_flushed;
}


static int io_rsrc_update_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
{
	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
		return -EINVAL;
	if (sqe->ioprio || sqe->rw_flags)
		return -EINVAL;

	req->rsrc_update.offset = READ_ONCE(sqe->off);
	req->rsrc_update.nr_args = READ_ONCE(sqe->len);
	if (!req->rsrc_update.nr_args)
		return -EINVAL;
	req->rsrc_update.arg = READ_ONCE(sqe->addr);
	return 0;


static int io_renameat_prep(struct io_kiocb *req,
			    const struct io_uring_sqe *sqe)
{
	struct io_rename *ren = &req->rename;
	const char __user *oldf, *newf;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;
	if (unlikely(req->flags & REQ_F_FIXED_FILE))
		return -EBADF;

	ren->old_dfd = READ_ONCE(sqe->fd);
	oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
	newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	ren->new_dfd = READ_ONCE(sqe->len);
	ren->flags = READ_ONCE(sqe->rename_flags);

	ren->oldpath = getname(oldf);
	if (IS_ERR(ren->oldpath))
		return PTR_ERR(ren->oldpath);

	ren->newpath = getname(newf);
	if (IS_ERR(ren->newpath)) {
		putname(ren->oldpath);
		return PTR_ERR(ren->newpath);
	}

	req->flags |= REQ_F_NEED_CLEANUP;
	return 0;
}


static void io_free_file_tables(struct io_file_table *table, unsigned nr_files)
{
	size_t size = nr_files * sizeof(struct io_fixed_file);

	io_free_page_table((void **)table->files, size);
	table->files = NULL;


static bool io_register_op_must_quiesce(int op)
{
	switch (op) {
	case IORING_REGISTER_BUFFERS:
	case IORING_UNREGISTER_BUFFERS:
	case IORING_REGISTER_FILES:
	case IORING_UNREGISTER_FILES:
	case IORING_REGISTER_FILES_UPDATE:
	case IORING_REGISTER_PROBE:
	case IORING_REGISTER_PERSONALITY:
	case IORING_UNREGISTER_PERSONALITY:
	case IORING_REGISTER_FILES2:
	case IORING_REGISTER_FILES_UPDATE2:
	case IORING_REGISTER_BUFFERS2:
	case IORING_REGISTER_BUFFERS_UPDATE:
	case IORING_REGISTER_IOWQ_AFF:
	case IORING_UNREGISTER_IOWQ_AFF:
		return false;
	default:
		return true;
	}


static void __io_queue_sqe(struct io_kiocb *req)
{
	struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
	int ret;

issue_sqe:
	ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);

	/*
	 * We async punt it if the file wasn't marked NOWAIT, or if the file
	 * doesn't support non-blocking read/write attempts
	 */
	if (likely(!ret)) {
		/* drop submission reference */
		if (req->flags & REQ_F_COMPLETE_INLINE) {
			struct io_ring_ctx *ctx = req->ctx;
			struct io_comp_state *cs = &ctx->submit_state.comp;

			cs->reqs[cs->nr++] = req;
			if (cs->nr == ARRAY_SIZE(cs->reqs))
				io_submit_flush_completions(ctx);
		} else {
			io_put_req(req);
		}
	} else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
		switch (io_arm_poll_handler(req)) {
		case IO_APOLL_READY:
			goto issue_sqe;
		case IO_APOLL_ABORTED:
			/*
			 * Queued up for async execution, worker will release
			 * submit reference when the iocb is actually submitted.
			 */
			io_queue_async_work(req);
			break;
		}
	} else {
		io_req_complete_failed(req, ret);
	}
	if (linked_timeout)
		io_queue_linked_timeout(linked_timeout);


static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
			    struct wait_queue_head *head,
			    struct io_poll_iocb **poll_ptr)
{
	struct io_kiocb *req = pt->req;

	/*
	 * If poll->head is already set, it's because the file being polled
	 * uses multiple waitqueues for poll handling (eg one for read, one
	 * for write). Setup a separate io_poll_iocb if this happens.
	 */
	if (unlikely(poll->head)) {
		struct io_poll_iocb *poll_one = poll;

		/* already have a 2nd entry, fail a third attempt */
		if (*poll_ptr) {
			pt->error = -EINVAL;
			return;
		}
		/*
		 * Can't handle multishot for double wait for now, turn it
		 * into one-shot mode.
		 */
		if (!(poll_one->events & EPOLLONESHOT))
			poll_one->events |= EPOLLONESHOT;
		/* double add on the same waitqueue head, ignore */
		if (poll_one->head == head)
			return;
		poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
		if (!poll) {
			pt->error = -ENOMEM;
			return;
		}
		io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake);
		req_ref_get(req);
		poll->wait.private = req;
		*poll_ptr = poll;
	}

	pt->error = 0;
	poll->head = head;

	if (poll->events & EPOLLEXCLUSIVE)
		add_wait_queue_exclusive(head, &poll->wait);
	else
		add_wait_queue(head, &poll->wait);


static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
{
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
	struct io_ring_ctx *ctx = req->ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
	list_del_init(&req->timeout.list);
	atomic_set(&req->ctx->cq_timeouts,
		atomic_read(&req->ctx->cq_timeouts) + 1);

	io_cqring_fill_event(ctx, req->user_data, -ETIME, 0);
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	io_cqring_ev_posted(ctx);
	req_set_fail(req);
	io_put_req(req);
	return HRTIMER_NORESTART;


static void io_tctx_exit_cb(struct callback_head *cb)
{
	struct io_uring_task *tctx = current->io_uring;
	struct io_tctx_exit *work;

	work = container_of(cb, struct io_tctx_exit, task_work);
	/*
	 * When @in_idle, we're in cancellation and it's racy to remove the
	 * node. It'll be removed by the end of cancellation, just ignore it.
	 */
	if (!atomic_read(&tctx->in_idle))
		io_uring_del_tctx_node((unsigned long)work->ctx);
	complete(&work->completion);


static void io_dismantle_req(struct io_kiocb *req)
{
	unsigned int flags = req->flags;

	if (io_req_needs_clean(req))
		io_clean_op(req);
	if (!(flags & REQ_F_FIXED_FILE))
		io_put_file(req->file);
	if (req->fixed_rsrc_refs)
		percpu_ref_put(req->fixed_rsrc_refs);
	if (req->async_data) {
		kfree(req->async_data);
		req->async_data = NULL;
	}
}


static void io_clean_op(struct io_kiocb *req)
{
	if (req->flags & REQ_F_BUFFER_SELECTED) {
		switch (req->opcode) {
		case IORING_OP_READV:
		case IORING_OP_READ_FIXED:
		case IORING_OP_READ:
			kfree((void *)(unsigned long)req->rw.addr);
			break;
		case IORING_OP_RECVMSG:
		case IORING_OP_RECV:
			kfree(req->sr_msg.kbuf);
			break;
		}
	}

	if (req->flags & REQ_F_NEED_CLEANUP) {
		switch (req->opcode) {
		case IORING_OP_READV:
		case IORING_OP_READ_FIXED:
		case IORING_OP_READ:
		case IORING_OP_WRITEV:
		case IORING_OP_WRITE_FIXED:
		case IORING_OP_WRITE: {
			struct io_async_rw *io = req->async_data;

			kfree(io->free_iovec);
			break;
			}
		case IORING_OP_RECVMSG:
		case IORING_OP_SENDMSG: {
			struct io_async_msghdr *io = req->async_data;

			kfree(io->free_iov);
			break;
			}
		case IORING_OP_SPLICE:
		case IORING_OP_TEE:
			if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED))
				io_put_file(req->splice.file_in);
			break;
		case IORING_OP_OPENAT:
		case IORING_OP_OPENAT2:
			if (req->open.filename)
				putname(req->open.filename);
			break;
		case IORING_OP_RENAMEAT:
			putname(req->rename.oldpath);
			putname(req->rename.newpath);
			break;
		case IORING_OP_UNLINKAT:
			putname(req->unlink.filename);
			break;
		}
	}
	if ((req->flags & REQ_F_POLLED) && req->apoll) {
		kfree(req->apoll->double_poll);
		kfree(req->apoll);
		req->apoll = NULL;
	}
	if (req->flags & REQ_F_INFLIGHT) {
		struct io_uring_task *tctx = req->task->io_uring;

		atomic_dec(&tctx->inflight_tracked);
	}
	if (req->flags & REQ_F_CREDS)
		put_cred(req->creds);

	req->flags &= ~IO_REQ_CLEAN_FLAGS;


static int io_uring_create(unsigned entries, struct io_uring_params *p,
			   struct io_uring_params __user *params)
{
	struct io_ring_ctx *ctx;
	struct file *file;
	int ret;

	if (!entries)
		return -EINVAL;
	if (entries > IORING_MAX_ENTRIES) {
		if (!(p->flags & IORING_SETUP_CLAMP))
			return -EINVAL;
		entries = IORING_MAX_ENTRIES;
	}

	/*
	 * Use twice as many entries for the CQ ring. It's possible for the
	 * application to drive a higher depth than the size of the SQ ring,
	 * since the sqes are only used at submission time. This allows for
	 * some flexibility in overcommitting a bit. If the application has
	 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
	 * of CQ ring entries manually.
	 */
	p->sq_entries = roundup_pow_of_two(entries);
	if (p->flags & IORING_SETUP_CQSIZE) {
		/*
		 * If IORING_SETUP_CQSIZE is set, we do the same roundup
		 * to a power-of-two, if it isn't already. We do NOT impose
		 * any cq vs sq ring sizing.
		 */
		if (!p->cq_entries)
			return -EINVAL;
		if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
			if (!(p->flags & IORING_SETUP_CLAMP))
				return -EINVAL;
			p->cq_entries = IORING_MAX_CQ_ENTRIES;
		}
		p->cq_entries = roundup_pow_of_two(p->cq_entries);
		if (p->cq_entries < p->sq_entries)
			return -EINVAL;
	} else {
		p->cq_entries = 2 * p->sq_entries;
	}

	ctx = io_ring_ctx_alloc(p);
	if (!ctx)
		return -ENOMEM;
	ctx->compat = in_compat_syscall();
	if (!capable(CAP_IPC_LOCK))
		ctx->user = get_uid(current_user());

	/*
	 * This is just grabbed for accounting purposes. When a process exits,
	 * the mm is exited and dropped before the files, hence we need to hang
	 * on to this mm purely for the purposes of being able to unaccount
	 * memory (locked/pinned vm). It's not used for anything else.
	 */
	mmgrab(current->mm);
	ctx->mm_account = current->mm;

	ret = io_allocate_scq_urings(ctx, p);
	if (ret)
		goto err;

	ret = io_sq_offload_create(ctx, p);
	if (ret)
		goto err;
	/* always set a rsrc node */
	ret = io_rsrc_node_switch_start(ctx);
	if (ret)
		goto err;
	io_rsrc_node_switch(ctx, NULL);

	memset(&p->sq_off, 0, sizeof(p->sq_off));
	p->sq_off.head = offsetof(struct io_rings, sq.head);
	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
	p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;

	memset(&p->cq_off, 0, sizeof(p->cq_off));
	p->cq_off.head = offsetof(struct io_rings, cq.head);
	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
	p->cq_off.cqes = offsetof(struct io_rings, cqes);
	p->cq_off.flags = offsetof(struct io_rings, cq_flags);

	p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
			IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
			IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
			IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
			IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
			IORING_FEAT_RSRC_TAGS;

	if (copy_to_user(params, p, sizeof(*p))) {
		ret = -EFAULT;
		goto err;
	}

	file = io_uring_get_file(ctx);
	if (IS_ERR(file)) {
		ret = PTR_ERR(file);
		goto err;
	}

	/*
	 * Install ring fd as the very last thing, so we don't risk someone
	 * having closed it before we finish setup
	 */
	ret = io_uring_install_fd(ctx, file);
	if (ret < 0) {
		/* fput will clean it up */
		fput(file);
		return ret;
	}

	trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
	return ret;
err:
	io_ring_ctx_wait_and_kill(ctx);
	return ret;


static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
{
	struct io_kiocb *nxt;

	/*
	 * If LINK is set, we have dependent requests in this chain. If we
	 * didn't fail this request, queue the first one up, moving any other
	 * dependencies to the next request. In case of failure, fail the rest
	 * of the chain.
	 */
	if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL)) {
		struct io_ring_ctx *ctx = req->ctx;
		unsigned long flags;
		bool posted;

		spin_lock_irqsave(&ctx->completion_lock, flags);
		posted = io_disarm_next(req);
		if (posted)
			io_commit_cqring(req->ctx);
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
		if (posted)
			io_cqring_ev_posted(ctx);
	}
	nxt = req->link;
	req->link = NULL;
	return nxt;
}


static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
{
	struct io_uring_task *tctx = current->io_uring;
	struct io_tctx_node *node;
	int ret;

	if (unlikely(!tctx)) {
		ret = io_uring_alloc_task_context(current, ctx);
		if (unlikely(ret))
			return ret;
		tctx = current->io_uring;
	}
	if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
		node = kmalloc(sizeof(*node), GFP_KERNEL);
		if (!node)
			return -ENOMEM;
		node->ctx = ctx;
		node->task = current;

		ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
					node, GFP_KERNEL));
		if (ret) {
			kfree(node);
			return ret;
		}

		mutex_lock(&ctx->uring_lock);
		list_add(&node->ctx_node, &ctx->tctx_list);
		mutex_unlock(&ctx->uring_lock);
	}
	tctx->last = ctx;
	return 0;