    std::string DebugString(const NodeDef& node) {
      std::string output;
      auto* ic = inference_context.get();
      absl::StrAppend(
          &output, node.name(), " [", node.op(), "] has ", ic->num_inputs(),
          (ic->num_inputs() > 1 ? " inputs and " : " input and "),
          ic->num_outputs(), (ic->num_outputs() > 1 ? " outputs" : " output"));
      if (op_data->is_function_op) {
        absl::StrAppend(&output, " (function op)");
      }
      absl::StrAppend(&output, ": \n");

      for (int i = 0; i < ic->num_inputs(); i++) {
        absl::StrAppend(&output, " input [", i, "] ", node.input(i),
                        " -- type: ", DataTypeString(input_types.at(i)),
                        ", shape: ", ic->DebugString(ic->input(i)),
                        ", tensor: ");
        Tensor t1;
        int input_tensor_protos_size = input_tensor_protos.size();
        if (input_tensor_protos_size > i &&
            input_tensor_protos.at(i) != nullptr &&
            t1.FromProto(*input_tensor_protos.at(i))) {
          absl::StrAppend(&output, t1.DebugString(), ", tensor_as_shape: ");
        } else {
          absl::StrAppend(&output, " null, tensor_as_shape: ");
        }
        int input_tensors_as_shapes_to_propagate_size =
            input_tensors_as_shapes_to_propagate.size();
        if (input_tensors_as_shapes_to_propagate_size > i) {
          absl::StrAppend(
              &output,
              StringifyShapeHandle(input_tensors_as_shapes_to_propagate.at(i)),
              "\n");
        } else {
          absl::StrAppend(&output, " null\n");
        }
      }
      for (int i = 0; i < ic->num_outputs(); i++) {
        absl::StrAppend(&output, " output [", i,
                        "] -- type: ", DataTypeString(output_types.at(i)),
                        ", shape: ", ic->DebugString(ic->output(i)),
                        ", tensor: ");
        Tensor t2;
        int output_tensor_protos_size = output_tensor_protos.size();
        if (output_tensor_protos_size > i &&
            output_tensor_protos.at(i) != nullptr &&
            t2.FromProto(*output_tensor_protos.at(i))) {
          absl::StrAppend(&output, t2.DebugString(), ", tensor_as_shape: ");
        } else {
          absl::StrAppend(&output, " null, tensor_as_shape: ");
        }
        int output_tensors_as_shapes_size = output_tensors_as_shapes.size();
        if (output_tensors_as_shapes_size > i) {
          absl::StrAppend(&output,
                          StringifyShapeHandle(output_tensors_as_shapes.at(i)),
                          "\n");
        } else {
          absl::StrAppend(&output, " null\n");
        }
      }
      return output;
    }


Prevent a crash due to heap OOB write in grappler.

PiperOrigin-RevId: 408318417
Change-Id: If095feb8c001e3a8ac4a85b7387b81e8309df47d

DisjointSet<Handle>::GetMergedValue(Handle value) {
  Rep* rep = Find(value);
  if (!rep) {
    // We don't know anything about this handle.
    return HandleToObject<Handle>::Unknown();
  }
  return rep->value;
}

Tensorflow is an Open Source Machine Learning Framework. TensorFlow is vulnerable to a heap OOB write in `Grappler`. The `set_output` function writes to an array at the specified index. Hence, this gives a malicious user a write primitive. The fix will be included in TensorFlow 2.8.0. We will also cherrypick this commit on TensorFlow 2.7.1, TensorFlow 2.6.3, and TensorFlow 2.5.3, as these are also affected and still in supported range.


  bool operator()(const Handle& h1, const Handle& h2) const {
    return h1.SameHandle(h2);
  }


void VerboseLogUnknownDimensionSources(
    const GraphDef& graph,
    const absl::flat_hash_map<string, std::vector<OpInfo::TensorProperties>>&
        input_properties_map,
    const absl::flat_hash_map<string, std::vector<OpInfo::TensorProperties>>&
        output_properties_map) {
  if (!VLOG_IS_ON(2)) {
    return;
  }

  VLOG(2) << "Nodes with known inputs, but with unknown output dimensions:";

  // Find all nodes in the graph for which we
  // do not have any unknown dimensions in their inputs, but
  // we have some unknown dimensions in their outputs.
  std::map<string, int> op_to_count;
  for (const NodeDef& node : graph.node()) {
    const auto& input_properties = input_properties_map.at(node.name());
    const auto& output_properties = output_properties_map.at(node.name());

    bool has_unknown_inputs = false;
    for (const auto& input_prop : input_properties) {
      if (HasAnyUnknownDimensions(input_prop.shape())) {
        has_unknown_inputs = true;
        break;
      }
    }

    if (has_unknown_inputs) {
      continue;
    }

    for (const auto& output_prop : output_properties) {
      if (HasAnyUnknownDimensions(output_prop.shape())) {
        string inputs = "input_shapes=[";
        for (const auto& input_prop : input_properties) {
          inputs += PartialTensorShape::DebugString(input_prop.shape());
        }
        inputs += "]";

        string outputs = "output_shapes=[";
        for (const auto& output_prop : output_properties) {
          outputs += PartialTensorShape::DebugString(output_prop.shape());
        }
        outputs += "]";

        VLOG(2) << "Node: " << node.name() << ", Op: " << node.op() << ", "
                << inputs << ", " << outputs;

        op_to_count[node.op()]++;

        // don't log again for this node
        break;
      }
    }
  }
  VLOG(2) << "Op types with known inputs, but with unknown output dimensions "
          << "(format: <op_type> (<count>)):";
  for (const auto& p : op_to_count) {
    VLOG(2) << p.first << " (" << p.second << ")";
  }
}


    std::string StringifyShapeHandle(ShapeHandle s) {
      auto* ic = inference_context.get();
      if (ic->RankKnown(s)) {
        std::vector<std::string> vals;
        for (int i = 0; i < ic->Rank(s); i++) {
          DimensionHandle d = ic->Dim(s, i);
          if (ic->ValueKnown(d) && ic->Value(d) == kUnknownDimFromConst) {
            vals.push_back("?(Const)");
          } else {
            vals.push_back(ic->DebugString(d));
          }
        }
        return strings::StrCat("[", absl::StrJoin(vals, ","), "]");
      } else {
        return "?";
      }
    }


  Status UpdateFunction(const NodeDef* function_node) {
    NameAttrList function;
    TF_RETURN_IF_ERROR(NameAndAttrsFromFunctionCall(*function_node, &function));
    auto it = fun_to_grappler_function_item_.find(function.name());
    if (it == fun_to_grappler_function_item_.end()) {
      return errors::InvalidArgument(
          function.name(),
          " was not previously added to SymbolicShapeRefiner.");
    }

    const absl::optional<GrapplerFunctionItem>& maybe_grappler_function_item =
        it->second;
    if (!maybe_grappler_function_item.has_value()) {
      VLOG(3) << "Skip failed to instantiate function call: function_name="
              << function.name();

      auto* ctx = GetNodeContext(function_node);
      auto* ic = ctx->inference_context.get();
      for (int i = 0; i < ic->num_outputs(); ++i) {
        TF_RETURN_IF_ERROR(SetUnknownShape(function_node, i));
      }

      return Status::OK();
    }

    // Copy (not reference) so that changes we make here (e.g., replacing
    // _Arg with Const and _Retval with Identity) don't affect one in
    // fun_to_grappler_function_item_.
    GrapplerFunctionItem grappler_function_item = *maybe_grappler_function_item;
    MutableGraphView gv(&grappler_function_item.graph);

    // Forward shapes from function input nodes to argument nodes.
    for (int i = 0, end = grappler_function_item.inputs().size(); i < end;
         ++i) {
      auto& fun_input = grappler_function_item.input(i);
      NodeDef* fun_node = gv.GetNode(fun_input.node_name);
      const TensorId input_tensor = ParseTensorName(function_node->input(i));

      if (IsControlInput(input_tensor)) {
        return errors::FailedPrecondition(
            "Function inputs should not contain control nodes.");
      }

      const NodeDef* input_node = graph_.GetNode(input_tensor.node());
      if (input_node == nullptr) {
        return errors::FailedPrecondition(input_tensor.node(),
                                          " was not found in the graph.");
      }

      InferenceContext* input_ic = GetContext(input_node);
      if (input_ic == nullptr) {
        return errors::FailedPrecondition(
            "Inference context has not been created for ", input_tensor.node());
      }

      int output_port_num = input_tensor.index();
      AttrValue attr_output_shape;
      TensorShapeProto proto;
      const auto handle = input_ic->output(output_port_num);
      input_ic->ShapeHandleToProto(handle, &proto);
      // There may be dim.size < -1 in SymbolicShapeRefiner. Change those to -1.
      NormalizeShapeForOutput(&proto);
      // _Arg op's output shape uses _output_shapes attr.
      AttrValue output_attr;
      output_attr.mutable_list()->add_shape()->Swap(&proto);
      (*fun_node->mutable_attr())["_output_shapes"] = output_attr;

      // If dtype is DT_RESOURCE, ops that read _Arg op use _handle_dtypes and
      // _handle_shapes attr for its shapes and dtypes.
      if (fun_input.data_type == DT_RESOURCE) {
        auto* shapes_and_types =
            input_ic->output_handle_shapes_and_types(output_port_num);
        if (shapes_and_types != nullptr && !shapes_and_types->empty()) {
          AttrValue dtype_attr;
          AttrValue shape_attr;
          for (const auto& shape_and_type : *shapes_and_types) {
            const auto& dtype = shape_and_type.dtype;
            const auto& shape_handle = shape_and_type.shape;
            dtype_attr.mutable_list()->add_type(dtype);
            input_ic->ShapeHandleToProto(
                shape_handle, shape_attr.mutable_list()->add_shape());
          }
          (*fun_node->mutable_attr())["_handle_dtypes"] = dtype_attr;
          (*fun_node->mutable_attr())["_handle_shapes"] = shape_attr;
        } else {
          // Note that we do not return error here, even if the input node does
          // not have shapes_and_types. Within the function, we cannot infer the
          // output shape of the DT_RESOURCE input; hence, potentially unknown
          // shapes/dims in the function output shapes.
          VLOG(2)
              << "A function node (" << function_node->name()
              << ") has input with DT_RESOURCE, but the input node does not "
              << "have shapes_and_types information: \n"
              << "function_node: " << function_node->ShortDebugString() << "\n"
              << "function input: " << i
              << ", input node's output: " << output_port_num << "\n"
              << "input node: " << input_node->ShortDebugString();
        }
      }
    }

    // ReplaceInputWithConst() may break GraphView's internal node mapping
    // structure; hence, we separately build node name to NodeDef* map, for the
    // output nodes (before GraphView becomes invalid). Note that we use string,
    // not string_view.
    absl::flat_hash_map<std::string, NodeDef*> output_nodes;
    for (const auto& output_arg : grappler_function_item.outputs()) {
      output_nodes[output_arg.node_name] = gv.GetNode(output_arg.node_name);
    }

    // Replace input nodes with Consts, if values are known. Note that
    // we don't check exceptions here as it's done in the above loop.
    auto* ctx = GetNodeContext(function_node);
    auto* ic = ctx->inference_context.get();
    for (int i = grappler_function_item.inputs().size() - 1; i >= 0; --i) {
      const string& input = function_node->input(i);
      const string node_name = NodeName(input);
      const NodeDef* input_node = graph_.GetNode(node_name);
      if (IsConstant(*input_node)) {
        TF_CHECK_OK(
            ReplaceInputWithConst(*input_node, i, &grappler_function_item));
      } else if (static_cast<int>(ctx->input_tensor_protos.size()) > i &&
                 ctx->input_tensor_protos[i] != nullptr) {
        NodeDef const_input_node = MakeConstNodeDefFromTensorProto(
            ic, *ctx->input_tensor_protos[i], ctx->input_types[i]);
        TF_CHECK_OK(ReplaceInputWithConst(const_input_node, i,
                                          &grappler_function_item));
      } else if (static_cast<int>(ic->input_tensors_as_shapes().size()) > i &&
                 IsShapeFullyDefinedIntegerVectorOrScalar(
                     ic, ic->input(i), ic->input_tensors_as_shapes()[i],
                     ctx->input_types[i])) {
        // We have fully defined input_tensors_as_shapes for this input; use it
        // as a const input to the function node.
        NodeDef const_input_node = MakeConstNodeDefFromShape(
            ic, ic->input(i), ic->input_tensors_as_shapes()[i],
            ctx->input_types[i]);
        TF_CHECK_OK(ReplaceInputWithConst(const_input_node, i,
                                          &grappler_function_item));
      }
    }
    // node_name to NodeDef* map in GraphView gv can be broken due to
    // ReplaceInputWithConst(). gv should not be used after this.

    // Replace output _Retval nodes with Identity nodes. _Retval is a system op
    // without outputs and registered shape function.
    for (const auto& output_arg : grappler_function_item.outputs()) {
      NodeDef* output_node = output_nodes[output_arg.node_name];
      DCHECK_EQ(output_node->op(), "_Retval");
      output_node->set_op("Identity");
      output_node->mutable_attr()->erase("index");
    }

    // Perform inference on function body.
    GraphProperties gp(grappler_function_item);
    TF_RETURN_IF_ERROR(gp.InferStatically(
        /*assume_valid_feeds=*/true,
        /*aggressive_shape_inference=*/aggressive_shape_inference_,
        /*include_tensor_values=*/true));

    // Add return nodes for output shapes.
    int output = 0;
    ctx->output_tensors_as_shapes.resize(grappler_function_item.output_size());
    ctx->output_tensor_protos.resize(grappler_function_item.output_size(),
                                     nullptr);
    for (auto const& out_arg : grappler_function_item.outputs()) {
      // It is guaranteed that output_tensors does not contain any control
      // inputs, so port_id >= 0.
      TensorId out_tensor = ParseTensorName(out_arg.node_name);

      if (output_nodes.count(out_tensor.node()) <= 0) {
        return errors::FailedPrecondition(
            "Unable to find return function_node ", out_tensor.node(), " for ",
            function_node->name());
      }
      const NodeDef* retnode = output_nodes[out_tensor.node()];

      auto output_properties = gp.GetOutputProperties(retnode->name());
      int output_properties_size = output_properties.size();
      if (out_tensor.index() >= output_properties_size) {
        return errors::InvalidArgument(
            out_tensor.ToString(), " has invalid position ", out_tensor.index(),
            " (output_properties.size() = ", output_properties.size(), ").");
      }
      auto& outprop = output_properties[out_tensor.index()];
      TensorShapeProto shape = outprop.shape();
      NormalizeShapeForOutput(&shape);
      ShapeHandle out;
      TF_RETURN_IF_ERROR(ic->MakeShapeFromShapeProto(shape, &out));
      ic->set_output(output, out);
      if (outprop.has_value()) {
        // Forward tensor value to output_tensors_as_shape.
        MaybeTensorProtoToShape(ic, outprop.value(),
                                &ctx->output_tensors_as_shapes[output]);
        const_tensors_to_propagate_.push_back(outprop.value());
        ctx->output_tensor_protos[output] = &const_tensors_to_propagate_.back();
      }
      output++;
    }

    return Status::OK();
  }


Eliminate debug `CHECK`-fail from `function.cc`

PiperOrigin-RevId: 409416119
Change-Id: I8376ee464d434e9b970ff0ad49edfdaa2a273cfe

  Status BuildInputArgIndex(const OpDef::ArgDef& arg_def, AttrSlice attr_values,
                            const FunctionDef::ArgAttrs* arg_attrs,
                            bool ints_on_device,
                            int64_t resource_arg_unique_id) {
    bool is_type_list;
    DataTypeVector dtypes;
    TF_RETURN_IF_ERROR(
        ArgNumType(attr_values, arg_def, &is_type_list, &dtypes));
    if (dtypes.size() < size_t{1}) {
      return errors::Internal("Expected a list of at least one dtype");
    }
    int arg_index = result_.nodes.size();
    TF_RETURN_IF_ERROR(
        AddItem(arg_def.name(), {true, arg_index, 0, is_type_list, dtypes}));
    // Creates dtypes.size() nodes in the graph.
    for (size_t i = 0; i < dtypes.size(); ++i) {
      TF_RETURN_IF_ERROR(AddItem(strings::StrCat(arg_def.name(), ":", i),
                                 {true, arg_index, 0, false, {dtypes[i]}}));
      if (arg_index != result_.nodes.size()) {
        return errors::Internal(
            "Expected arg_index to be equal to the number of nodes in result.",
            " Got ", arg_index, " and ", result_.nodes.size());
      }
      string name = arg_def.name();
      if (dtypes.size() > 1) {
        strings::StrAppend(&name, "_", i);
      }
      NodeDef* gnode = AddNode(name);
      if (ints_on_device && dtypes[i] == DataType::DT_INT32) {
        gnode->set_op(FunctionLibraryDefinition::kDeviceArgOp);
      } else {
        gnode->set_op(FunctionLibraryDefinition::kArgOp);
      }
      DataType dtype = arg_def.is_ref() ? MakeRefType(dtypes[i]) : dtypes[i];
      AddAttr("T", dtype, gnode);
      AddAttr("index", arg_index, gnode);
      if (resource_arg_unique_id >= 0) {
        AddAttr("_resource_arg_unique_id", resource_arg_unique_id, gnode);
      }
      if (arg_attrs) {
        for (const auto& arg_attr : arg_attrs->attr()) {
          AddAttr(arg_attr.first, arg_attr.second, gnode->mutable_attr());
        }
      }
      result_.arg_types.push_back(dtypes[i]);
      ++arg_index;
    }
    return Status::OK();
  }

Tensorflow is an Open Source Machine Learning Framework. A malicious user can cause a denial of service by altering a `SavedModel` such that assertions in `function.cc` would be falsified and crash the Python interpreter. The fix will be included in TensorFlow 2.8.0. We will also cherrypick this commit on TensorFlow 2.7.1, TensorFlow 2.6.3, and TensorFlow 2.5.3, as these are also affected and still in supported range.


Fixes #4985: 4e+26 is outside the range of representable values of type 'unsigned long' at coders/pcl.c:299 (#4986)

* fix Division by zero in XMenuWidget() of MagickCore/widget.c

* Fix memory leak in AnimateImageCommand() of MagickWand/animate.c and DisplayImageCommand() of MagickWand/display.c

* fix Division by zero in ReadEnhMetaFile() of coders/emf.c

* Resolve conflicts

* fix issue: outside the range of representable values of type 'unsigned char' at coders/psd.c:1025

* fix error: 4e+26 is outside the range of representable values of type 'unsigned long' at coders/pcl.c:299

Co-authored-by: zhailiangliang <zhailiangliang@loongson.cn>

ModuleExport void UnregisterPCLImage(void)
{
  (void) UnregisterMagickInfo("PCL");
}

A vulnerability was found in ImageMagick, causing an outside the range of representable values of type 'unsigned long' at coders/pcl.c, when crafted or untrusted input is processed. This leads to a negative impact to application availability or other problems related to undefined behavior.


static Image *ReadPCLImage(const ImageInfo *image_info,ExceptionInfo *exception)
{
#define CropBox  "CropBox"
#define DeviceCMYK  "DeviceCMYK"
#define MediaBox  "MediaBox"
#define RenderPCLText  "  Rendering PCL...  "

  char
    command[MagickPathExtent],
    *density,
    filename[MagickPathExtent],
    geometry[MagickPathExtent],
    *options,
    input_filename[MagickPathExtent];

  const DelegateInfo
    *delegate_info;

  Image
    *image,
    *next_image;

  ImageInfo
    *read_info;

  MagickBooleanType
    cmyk,
    status;

  PointInfo
    delta;

  RectangleInfo
    bounding_box,
    page;

  char
    *p;

  ssize_t
    c;

  SegmentInfo
    bounds;

  size_t
    height,
    width;

  ssize_t
    count;

  assert(image_info != (const ImageInfo *) NULL);
  assert(image_info->signature == MagickCoreSignature);
  if (image_info->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",
      image_info->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  /*
    Open image file.
  */
  image=AcquireImage(image_info,exception);
  status=OpenBlob(image_info,image,ReadBinaryBlobMode,exception);
  if (status == MagickFalse)
    {
      image=DestroyImageList(image);
      return((Image *) NULL);
    }
  status=AcquireUniqueSymbolicLink(image_info->filename,input_filename);
  if (status == MagickFalse)
    {
      ThrowFileException(exception,FileOpenError,"UnableToCreateTemporaryFile",
        image_info->filename);
      image=DestroyImageList(image);
      return((Image *) NULL);
    }
  /*
    Set the page density.
  */
  delta.x=DefaultResolution;
  delta.y=DefaultResolution;
  if ((image->resolution.x == 0.0) || (image->resolution.y == 0.0))
    {
      GeometryInfo
        geometry_info;

      MagickStatusType
        flags;

      flags=ParseGeometry(PSDensityGeometry,&geometry_info);
      if ((flags & RhoValue) != 0)
        image->resolution.x=geometry_info.rho;
      image->resolution.y=image->resolution.x;
      if ((flags & SigmaValue) != 0)
        image->resolution.y=geometry_info.sigma;
    }
  /*
    Determine page geometry from the PCL media box.
  */
  cmyk=image->colorspace == CMYKColorspace ? MagickTrue : MagickFalse;
  count=0;
  (void) memset(&bounding_box,0,sizeof(bounding_box));
  (void) memset(&bounds,0,sizeof(bounds));
  (void) memset(&page,0,sizeof(page));
  (void) memset(command,0,sizeof(command));
  p=command;
  for (c=ReadBlobByte(image); c != EOF; c=ReadBlobByte(image))
  {
    if (image_info->page != (char *) NULL)
      continue;
    /*
      Note PCL elements.
    */
    *p++=(char) c;
    if ((c != (int) '/') && (c != '\n') &&
        ((size_t) (p-command) < (MagickPathExtent-1)))
      continue;
    *p='\0';
    p=command;
    /*
      Is this a CMYK document?
    */
    if (LocaleNCompare(DeviceCMYK,command,strlen(DeviceCMYK)) == 0)
      cmyk=MagickTrue;
    if (LocaleNCompare(CropBox,command,strlen(CropBox)) == 0)
      {
        /*
          Note region defined by crop box.
        */
        count=(ssize_t) sscanf(command,"CropBox [%lf %lf %lf %lf",
          &bounds.x1,&bounds.y1,&bounds.x2,&bounds.y2);
        if (count != 4)
          count=(ssize_t) sscanf(command,"CropBox[%lf %lf %lf %lf",
            &bounds.x1,&bounds.y1,&bounds.x2,&bounds.y2);
      }
    if (LocaleNCompare(MediaBox,command,strlen(MediaBox)) == 0)
      {
        /*
          Note region defined by media box.
        */
        count=(ssize_t) sscanf(command,"MediaBox [%lf %lf %lf %lf",
          &bounds.x1,&bounds.y1,&bounds.x2,&bounds.y2);
        if (count != 4)
          count=(ssize_t) sscanf(command,"MediaBox[%lf %lf %lf %lf",
            &bounds.x1,&bounds.y1,&bounds.x2,&bounds.y2);
      }
    if (count != 4)
      continue;
    /*
      Set PCL render geometry.
    */
    width=(size_t)CastDoubleToLong(floor(bounds.x2-bounds.x1+0.5));
    height=(size_t)CastDoubleToLong(floor(bounds.y2-bounds.y1+0.5));
    if (width > page.width)
      page.width=width;
    if (height > page.height)
      page.height=height;
  }
  (void) CloseBlob(image);
  /*
    Render PCL with the GhostPCL delegate.
  */
  if ((page.width == 0) || (page.height == 0))
    (void) ParseAbsoluteGeometry(PSPageGeometry,&page);
  if (image_info->page != (char *) NULL)
    (void) ParseAbsoluteGeometry(image_info->page,&page);
  (void) FormatLocaleString(geometry,MagickPathExtent,"%.20gx%.20g",(double)
    page.width,(double) page.height);
  if (image_info->monochrome != MagickFalse)
    delegate_info=GetDelegateInfo("pcl:mono",(char *) NULL,exception);
  else
     if (cmyk != MagickFalse)
       delegate_info=GetDelegateInfo("pcl:cmyk",(char *) NULL,exception);
     else
       delegate_info=GetDelegateInfo("pcl:color",(char *) NULL,exception);
  if (delegate_info == (const DelegateInfo *) NULL)
    {
      image=DestroyImage(image);
      return((Image *) NULL);
    }
  if ((page.width == 0) || (page.height == 0))
    (void) ParseAbsoluteGeometry(PSPageGeometry,&page);
  if (image_info->page != (char *) NULL)
    (void) ParseAbsoluteGeometry(image_info->page,&page);
  density=AcquireString("");
  options=AcquireString("");
  (void) FormatLocaleString(density,MagickPathExtent,"%gx%g",
    image->resolution.x,image->resolution.y);
  if (image_info->ping != MagickFalse)
    (void) FormatLocaleString(density,MagickPathExtent,"2.0x2.0");
  page.width=(size_t) floor(page.width*image->resolution.x/delta.x+0.5);
  page.height=(size_t) floor(page.height*image->resolution.y/delta.y+0.5);
  (void) FormatLocaleString(options,MagickPathExtent,"-g%.20gx%.20g ",(double)
    page.width,(double) page.height);
  image=DestroyImage(image);
  read_info=CloneImageInfo(image_info);
  *read_info->magick='\0';
  if (read_info->number_scenes != 0)
    {
      if (read_info->number_scenes != 1)
        (void) FormatLocaleString(options,MagickPathExtent,"-dLastPage=%.20g",
          (double) (read_info->scene+read_info->number_scenes));
      else
        (void) FormatLocaleString(options,MagickPathExtent,
          "-dFirstPage=%.20g -dLastPage=%.20g",(double) read_info->scene+1,
          (double) (read_info->scene+read_info->number_scenes));
      read_info->number_scenes=0;
      if (read_info->scenes != (char *) NULL)
        *read_info->scenes='\0';
    }
  (void) CopyMagickString(filename,read_info->filename,MagickPathExtent);
  (void) AcquireUniqueFilename(read_info->filename);
  (void) FormatLocaleString(command,MagickPathExtent,
    GetDelegateCommands(delegate_info),
    read_info->antialias != MagickFalse ? 4 : 1,
    read_info->antialias != MagickFalse ? 4 : 1,density,options,
    read_info->filename,input_filename);
  options=DestroyString(options);
  density=DestroyString(density);
  status=ExternalDelegateCommand(MagickFalse,read_info->verbose,command,
    (char *) NULL,exception) != 0 ? MagickTrue : MagickFalse;
  image=ReadImage(read_info,exception);
  (void) RelinquishUniqueFileResource(read_info->filename);
  (void) RelinquishUniqueFileResource(input_filename);
  read_info=DestroyImageInfo(read_info);
  if (image == (Image *) NULL)
    ThrowReaderException(DelegateError,"PCLDelegateFailed");
  if (LocaleCompare(image->magick,"BMP") == 0)
    {
      Image
        *cmyk_image;

      cmyk_image=ConsolidateCMYKImages(image,exception);
      if (cmyk_image != (Image *) NULL)
        {
          image=DestroyImageList(image);
          image=cmyk_image;
        }
    }
  do
  {
    (void) CopyMagickString(image->filename,filename,MagickPathExtent);
    image->page=page;
    if (image_info->ping != MagickFalse)
      {
        image->magick_columns*=image->resolution.x/2.0;
        image->magick_rows*=image->resolution.y/2.0;
        image->columns*=image->resolution.x/2.0;
        image->rows*=image->resolution.y/2.0;
      }
    next_image=SyncNextImageInList(image);
    if (next_image != (Image *) NULL)
      image=next_image;
  } while (next_image != (Image *) NULL);
  return(GetFirstImageInList(image));
}


static size_t PCLPackbitsCompressImage(const size_t length,
  const unsigned char *pixels,unsigned char *compress_pixels)
{
  int
    count;

  ssize_t
    x;

  unsigned char
    *q;

  ssize_t
    j;

  unsigned char
    packbits[128];

  /*
    Compress pixels with Packbits encoding.
  */
  q=compress_pixels;
  for (x=(ssize_t) length; x != 0; )
  {
    switch (x)
    {
      case 1:
      {
        x--;
        *q++=0;
        *q++=(*pixels);
        break;
      }
      case 2:
      {
        x-=2;
        *q++=1;
        *q++=(*pixels);
        *q++=pixels[1];
        break;
      }
      case 3:
      {
        x-=3;
        if ((*pixels == *(pixels+1)) && (*(pixels+1) == *(pixels+2)))
          {
            *q++=(unsigned char) ((256-3)+1);
            *q++=(*pixels);
            break;
          }
        *q++=2;
        *q++=(*pixels);
        *q++=pixels[1];
        *q++=pixels[2];
        break;
      }
      default:
      {
        if ((*pixels == *(pixels+1)) && (*(pixels+1) == *(pixels+2)))
          {
            /*
              Packed run.
            */
            count=3;
            while (((ssize_t) count < x) && (*pixels == *(pixels+count)))
            {
              count++;
              if (count >= 127)
                break;
            }
            x-=count;
            *q++=(unsigned char) ((256-count)+1);
            *q++=(*pixels);
            pixels+=count;
            break;
          }
        /*
          Literal run.
        */
        count=0;
        while ((*(pixels+count) != *(pixels+count+1)) ||
               (*(pixels+count+1) != *(pixels+count+2)))
        {
          packbits[count+1]=pixels[count];
          count++;
          if (((ssize_t) count >= (x-3)) || (count >= 127))
            break;
        }
        x-=count;
        *packbits=(unsigned char) (count-1);
        for (j=0; j <= (ssize_t) count; j++)
          *q++=packbits[j];
        pixels+=count;
        break;
      }
    }
  }
  *q++=128; /* EOD marker */
  return((size_t) (q-compress_pixels));
}


ModuleExport size_t RegisterPCLImage(void)
{
  MagickInfo
    *entry;

  entry=AcquireMagickInfo("PCL","PCL","Printer Control Language");
  entry->decoder=(DecodeImageHandler *) ReadPCLImage;
  entry->encoder=(EncodeImageHandler *) WritePCLImage;
  entry->magick=(IsImageFormatHandler *) IsPCL;
  entry->flags^=CoderBlobSupportFlag;
  entry->flags^=CoderDecoderThreadSupportFlag;
  (void) RegisterMagickInfo(entry);
  return(MagickImageCoderSignature);
}


static MagickBooleanType WritePCLImage(const ImageInfo *image_info,Image *image,
  ExceptionInfo *exception)
{
  char
    buffer[MagickPathExtent];

  CompressionType
    compression;

  const char
    *option;

  MagickBooleanType
    status;

  MagickOffsetType
    scene;

  const Quantum *p;

  ssize_t i, x;

  unsigned char *q;

  size_t
    density,
    imageListLength,
    length,
    one,
    packets;

  ssize_t
    y;

  unsigned char
    bits_per_pixel,
    *compress_pixels,
    *pixels,
    *previous_pixels;

  /*
    Open output image file.
  */
  assert(image_info != (const ImageInfo *) NULL);
  assert(image_info->signature == MagickCoreSignature);
  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  status=OpenBlob(image_info,image,WriteBinaryBlobMode,exception);
  if (status == MagickFalse)
    return(status);
  density=75;
  if (image_info->density != (char *) NULL)
    {
      GeometryInfo
        geometry;

      (void) ParseGeometry(image_info->density,&geometry);
      density=(size_t) geometry.rho;
    }
  scene=0;
  one=1;
  imageListLength=GetImageListLength(image);
  do
  {
    /*
      Initialize the printer.
    */
    if (IssRGBCompatibleColorspace(image->colorspace) == MagickFalse)
      (void) TransformImageColorspace(image,sRGBColorspace,exception);
    (void) WriteBlobString(image,"\033E");  /* printer reset */
    (void) WriteBlobString(image,"\033*r3F");  /* set presentation mode */
    (void) FormatLocaleString(buffer,MagickPathExtent,"\033*r%.20gs%.20gT",
      (double) image->columns,(double) image->rows);
    (void) WriteBlobString(image,buffer);
    (void) FormatLocaleString(buffer,MagickPathExtent,"\033*t%.20gR",(double)
      density);
    (void) WriteBlobString(image,buffer);
    (void) WriteBlobString(image,"\033&l0E");  /* top margin 0 */
    if (SetImageMonochrome(image,exception) != MagickFalse)
      {
        /*
          Monochrome image: use default printer monochrome setup.
        */
        bits_per_pixel=1;
      }
    else
      if (image->storage_class == DirectClass)
        {
          /*
            DirectClass image.
          */
          bits_per_pixel=24;
          (void) WriteBlobString(image,"\033*v6W"); /* set color mode */
          (void) WriteBlobByte(image,0); /* RGB */
          (void) WriteBlobByte(image,3); /* direct by pixel */
          (void) WriteBlobByte(image,0); /* bits per index (ignored) */
          (void) WriteBlobByte(image,8); /* bits per red component */
          (void) WriteBlobByte(image,8); /* bits per green component */
          (void) WriteBlobByte(image,8); /* bits per blue component */
        }
      else
        {
          /*
            Colormapped image.
          */
          bits_per_pixel=8;
          (void) WriteBlobString(image,"\033*v6W"); /* set color mode... */
          (void) WriteBlobByte(image,0); /* RGB */
          (void) WriteBlobByte(image,1); /* indexed by pixel */
          (void) WriteBlobByte(image,bits_per_pixel); /* bits per index */
          (void) WriteBlobByte(image,8); /* bits per red component */
          (void) WriteBlobByte(image,8); /* bits per green component */
          (void) WriteBlobByte(image,8); /* bits per blue component */
          for (i=0; i < (ssize_t) image->colors; i++)
          {
            (void) FormatLocaleString(buffer,MagickPathExtent,
              "\033*v%da%db%dc%.20gI",
              ScaleQuantumToChar(image->colormap[i].red),
              ScaleQuantumToChar(image->colormap[i].green),
              ScaleQuantumToChar(image->colormap[i].blue),(double) i);
            (void) WriteBlobString(image,buffer);
          }
          for (one=1; i < (ssize_t) (one << bits_per_pixel); i++)
          {
            (void) FormatLocaleString(buffer,MagickPathExtent,"\033*v%.20gI",
              (double) i);
            (void) WriteBlobString(image,buffer);
          }
        }
    option=GetImageOption(image_info,"pcl:fit-to-page");
    if (IsStringTrue(option) != MagickFalse)
      (void) WriteBlobString(image,"\033*r3A");
    else
      (void) WriteBlobString(image,"\033*r1A");  /* start raster graphics */
    (void) WriteBlobString(image,"\033*b0Y");  /* set y offset */
    length=(image->columns*bits_per_pixel+7)/8;
    pixels=(unsigned char *) AcquireQuantumMemory(length+1,sizeof(*pixels));
    if (pixels == (unsigned char *) NULL)
      ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
    (void) memset(pixels,0,(length+1)*sizeof(*pixels));
    compress_pixels=(unsigned char *) NULL;
    previous_pixels=(unsigned char *) NULL;

    compression=UndefinedCompression;
    if (image_info->compression != UndefinedCompression)
      compression=image_info->compression;
    switch (compression)
    {
      case NoCompression:
      {
        (void) FormatLocaleString(buffer,MagickPathExtent,"\033*b0M");
        (void) WriteBlobString(image,buffer);
        break;
      }
      case RLECompression:
      {
        compress_pixels=(unsigned char *) AcquireQuantumMemory(length+256,
          sizeof(*compress_pixels));
        if (compress_pixels == (unsigned char *) NULL)
          {
            pixels=(unsigned char *) RelinquishMagickMemory(pixels);
            ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
          }
        (void) memset(compress_pixels,0,(length+256)*
          sizeof(*compress_pixels));
        (void) FormatLocaleString(buffer,MagickPathExtent,"\033*b2M");
        (void) WriteBlobString(image,buffer);
        break;
      }
      default:
      {
        compress_pixels=(unsigned char *) AcquireQuantumMemory(3*length+256,
          sizeof(*compress_pixels));
        if (compress_pixels == (unsigned char *) NULL)
          {
            pixels=(unsigned char *) RelinquishMagickMemory(pixels);
            ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
          }
        (void) memset(compress_pixels,0,(3*length+256)*
          sizeof(*compress_pixels));
        previous_pixels=(unsigned char *) AcquireQuantumMemory(length+1,
          sizeof(*previous_pixels));
        if (previous_pixels == (unsigned char *) NULL)
          {
            compress_pixels=(unsigned char *) RelinquishMagickMemory(
              compress_pixels);
            pixels=(unsigned char *) RelinquishMagickMemory(pixels);
            ThrowWriterException(ResourceLimitError,"MemoryAllocationFailed");
          }
        (void) memset(previous_pixels,0,(length+1)*
          sizeof(*previous_pixels));
        (void) FormatLocaleString(buffer,MagickPathExtent,"\033*b3M");
        (void) WriteBlobString(image,buffer);
        break;
      }
    }
    for (y=0; y < (ssize_t) image->rows; y++)
    {
      p=GetVirtualPixels(image,0,y,image->columns,1,exception);
      if (p == (const Quantum *) NULL)
        break;
      q=pixels;
      switch (bits_per_pixel)
      {
        case 1:
        {
          unsigned char
            bit,
            byte;

          /*
            Monochrome image.
          */
          bit=0;
          byte=0;
          for (x=0; x < (ssize_t) image->columns; x++)
          {
            byte<<=1;
            if (GetPixelLuma(image,p) < (QuantumRange/2.0))
              byte|=0x01;
            bit++;
            if (bit == 8)
              {
                *q++=byte;
                bit=0;
                byte=0;
              }
            p+=GetPixelChannels(image);
          }
          if (bit != 0)
            *q++=byte << (8-bit);
          break;
        }
        case 8:
        {
          /*
            Colormapped image.
          */
          for (x=0; x < (ssize_t) image->columns; x++)
          {
            *q++=(unsigned char) ((ssize_t) GetPixelIndex(image,p));
            p+=GetPixelChannels(image);
          }
          break;
        }
        case 24:
        case 32:
        {
          /*
            Truecolor image.
          */
          for (x=0; x < (ssize_t) image->columns; x++)
          {
            *q++=ScaleQuantumToChar(GetPixelRed(image,p));
            *q++=ScaleQuantumToChar(GetPixelGreen(image,p));
            *q++=ScaleQuantumToChar(GetPixelBlue(image,p));
            p+=GetPixelChannels(image);
          }
          break;
        }
      }
      switch (compression)
      {
        case NoCompression:
        {
          (void) FormatLocaleString(buffer,MagickPathExtent,"\033*b%.20gW",
            (double) length);
          (void) WriteBlobString(image,buffer);
          (void) WriteBlob(image,length,pixels);
          break;
        }
        case RLECompression:
        {
          packets=PCLPackbitsCompressImage(length,pixels,compress_pixels);
          (void) FormatLocaleString(buffer,MagickPathExtent,"\033*b%.20gW",
            (double) packets);
          (void) WriteBlobString(image,buffer);
          (void) WriteBlob(image,packets,compress_pixels);
          break;
        }
        default:
        {
          if (y == 0)
            for (i=0; i < (ssize_t) length; i++)
              previous_pixels[i]=(~pixels[i]);
          packets=PCLDeltaCompressImage(length,previous_pixels,pixels,
            compress_pixels);
          (void) FormatLocaleString(buffer,MagickPathExtent,"\033*b%.20gW",
            (double) packets);
          (void) WriteBlobString(image,buffer);
          (void) WriteBlob(image,packets,compress_pixels);
          (void) memcpy(previous_pixels,pixels,length*
            sizeof(*pixels));
          break;
        }
      }
    }
    (void) WriteBlobString(image,"\033*rB");  /* end graphics */
    switch (compression)
    {
      case NoCompression:
        break;
      case RLECompression:
      {
        compress_pixels=(unsigned char *) RelinquishMagickMemory(
          compress_pixels);
        break;
      }
      default:
      {
        previous_pixels=(unsigned char *) RelinquishMagickMemory(
          previous_pixels);
        compress_pixels=(unsigned char *) RelinquishMagickMemory(
          compress_pixels);
        break;
      }
    }
    pixels=(unsigned char *) RelinquishMagickMemory(pixels);
    if (GetNextImageInList(image) == (Image *) NULL)
      break;
    image=SyncNextImageInList(image);
    status=SetImageProgress(image,SaveImagesTag,scene++,imageListLength);
    if (status == MagickFalse)
      break;
  } while (image_info->adjoin != MagickFalse);
  (void) WriteBlobString(image,"\033E");
  (void) CloseBlob(image);
  return(MagickTrue);
}


static size_t PCLDeltaCompressImage(const size_t length,
  const unsigned char *previous_pixels,const unsigned char *pixels,
  unsigned char *compress_pixels)
{
  int
    delta,
    j,
    replacement;

  ssize_t
    i,
    x;

  unsigned char
    *q;

  q=compress_pixels;
  for (x=0; x < (ssize_t) length; )
  {
    j=0;
    for (i=0; x < (ssize_t) length; x++)
    {
      if (*pixels++ != *previous_pixels++)
        {
          i=1;
          break;
        }
      j++;
    }
    while (x < (ssize_t) length)
    {
      x++;
      if (*pixels == *previous_pixels)
        break;
      i++;
      previous_pixels++;
      pixels++;
    }
    if (i == 0)
      break;
    replacement=j >= 31 ? 31 : j;
    j-=replacement;
    delta=i >= 8 ? 8 : i;
    *q++=(unsigned char) (((delta-1) << 5) | replacement);
    if (replacement == 31)
      {
        for (replacement=255; j != 0; )
        {
          if (replacement > j)
            replacement=j;
          *q++=(unsigned char) replacement;
          j-=replacement;
        }
        if (replacement == 255)
          *q++='\0';
      }
    for (pixels-=i; i != 0; )
    {
      for (i-=delta; delta != 0; delta--)
        *q++=(*pixels++);
      if (i == 0)
        break;
      delta=i;
      if (i >= 8)
        delta=8;
      *q++=(unsigned char) ((delta-1) << 5);
    }
  }
  return((size_t) (q-compress_pixels));
}


static MagickBooleanType IsPCL(const unsigned char *magick,const size_t length)
{
  if (length < 4)
    return(MagickFalse);
  if (memcmp(magick,"\033E\033&",4) == 0)
    return(MagickFalse);
  if (memcmp(magick,"\033E\033",3) == 0)
    return(MagickTrue);
  return(MagickFalse);
}


Add missing validation to sparse dense cwise ops.

PiperOrigin-RevId: 415543133
Change-Id: I5baf3284e919338afb96178c468ad3d3cb0d956c

  explicit SparseDenseBinaryOpShared(OpKernelConstruction *ctx)
      : OpKernel(ctx) {}

Tensorflow is an Open Source Machine Learning Framework. The implementations of `Sparse*Cwise*` ops are vulnerable to integer overflows. These can be used to trigger large allocations (so, OOM based denial of service) or `CHECK`-fails when building new `TensorShape` objects (so, assert failures based denial of service). We are missing some validation on the shapes of the input tensors as well as directly constructing a large `TensorShape` with user-provided dimensions. The fix will be included in TensorFlow 2.8.0. We will also cherrypick this commit on TensorFlow 2.7.1, TensorFlow 2.6.3, and TensorFlow 2.5.3, as these are also affected and still in supported range.


static PCRE2_SPTR bracketend(PCRE2_SPTR cc)
{
SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NA) || (*cc >= OP_ONCE && *cc <= OP_SCOND));
do cc += GET(cc, 1); while (*cc == OP_ALT);
SLJIT_ASSERT(*cc >= OP_KET && *cc <= OP_KETRPOS);
cc += 1 + LINK_SIZE;
return cc;
}

An out-of-bounds read vulnerability was discovered in the PCRE2 library in the compile_xclass_matchingpath() function of the pcre2_jit_compile.c file. This involves a unicode property matching issue in JIT-compiled regular expressions. The issue occurs because the character was not fully read in case-less matching within JIT.


static SLJIT_INLINE void compile_recurse(compiler_common *common)
{
DEFINE_COMPILER;
PCRE2_SPTR cc = common->start + common->currententry->start;
PCRE2_SPTR ccbegin = cc + 1 + LINK_SIZE + (*cc == OP_BRA ? 0 : IMM2_SIZE);
PCRE2_SPTR ccend = bracketend(cc) - (1 + LINK_SIZE);
BOOL needs_control_head;
BOOL has_quit;
BOOL has_accept;
int private_data_size = get_recurse_data_length(common, ccbegin, ccend, &needs_control_head, &has_quit, &has_accept);
int alt_count, alt_max, local_size;
backtrack_common altbacktrack;
jump_list *match = NULL;
struct sljit_jump *next_alt = NULL;
struct sljit_jump *accept_exit = NULL;
struct sljit_label *quit;
struct sljit_put_label *put_label = NULL;

/* Recurse captures then. */
common->then_trap = NULL;

SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA || *cc == OP_CBRAPOS || *cc == OP_SCBRA || *cc == OP_SCBRAPOS);

alt_max = no_alternatives(cc);
alt_count = 0;

/* Matching path. */
SLJIT_ASSERT(common->currententry->entry_label == NULL && common->recursive_head_ptr != 0);
common->currententry->entry_label = LABEL();
set_jumps(common->currententry->entry_calls, common->currententry->entry_label);

sljit_emit_fast_enter(compiler, TMP2, 0);
count_match(common);

local_size = (alt_max > 1) ? 2 : 1;

/* (Reversed) stack layout:
   [private data][return address][optional: str ptr] ... [optional: alternative index][recursive_head_ptr] */

allocate_stack(common, private_data_size + local_size);
/* Save return address. */
OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(local_size - 1), TMP2, 0);

copy_recurse_data(common, ccbegin, ccend, recurse_copy_from_global, local_size, private_data_size + local_size, has_quit);

/* This variable is saved and restored all time when we enter or exit from a recursive context. */
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, STACK_TOP, 0);

if (needs_control_head)
  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0);

if (alt_max > 1)
  OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);

memset(&altbacktrack, 0, sizeof(backtrack_common));
common->quit_label = NULL;
common->accept_label = NULL;
common->quit = NULL;
common->accept = NULL;
altbacktrack.cc = ccbegin;
cc += GET(cc, 1);
while (1)
  {
  altbacktrack.top = NULL;
  altbacktrack.topbacktracks = NULL;

  if (altbacktrack.cc != ccbegin)
    OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));

  compile_matchingpath(common, altbacktrack.cc, cc, &altbacktrack);
  if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
    return;

  allocate_stack(common, (alt_max > 1 || has_accept) ? 2 : 1);
  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr);

  if (alt_max > 1 || has_accept)
    {
    if (alt_max > 3)
      put_label = sljit_emit_put_label(compiler, SLJIT_MEM1(STACK_TOP), STACK(1));
    else
      OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, alt_count);
    }

  add_jump(compiler, &match, JUMP(SLJIT_JUMP));

  if (alt_count == 0)
    {
    /* Backtracking path entry. */
    SLJIT_ASSERT(common->currententry->backtrack_label == NULL);
    common->currententry->backtrack_label = LABEL();
    set_jumps(common->currententry->backtrack_calls, common->currententry->backtrack_label);

    sljit_emit_fast_enter(compiler, TMP1, 0);

    if (has_accept)
      accept_exit = CMP(SLJIT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, -1);

    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
    /* Save return address. */
    OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), STACK(local_size - 1), TMP1, 0);

    copy_recurse_data(common, ccbegin, ccend, recurse_swap_global, local_size, private_data_size + local_size, has_quit);

    if (alt_max > 1)
      {
      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1));
      free_stack(common, 2);

      if (alt_max > 3)
        {
        sljit_emit_ijump(compiler, SLJIT_JUMP, TMP1, 0);
        sljit_set_put_label(put_label, LABEL());
        sljit_emit_op0(compiler, SLJIT_ENDBR);
        }
      else
        next_alt = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0);
      }
    else
      free_stack(common, has_accept ? 2 : 1);
    }
  else if (alt_max > 3)
    {
    sljit_set_put_label(put_label, LABEL());
    sljit_emit_op0(compiler, SLJIT_ENDBR);
    }
  else
    {
    JUMPHERE(next_alt);
    if (alt_count + 1 < alt_max)
      {
      SLJIT_ASSERT(alt_count == 1 && alt_max == 3);
      next_alt = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 1);
      }
    }

  alt_count++;

  compile_backtrackingpath(common, altbacktrack.top);
  if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
    return;
  set_jumps(altbacktrack.topbacktracks, LABEL());

  if (*cc != OP_ALT)
    break;

  altbacktrack.cc = cc + 1 + LINK_SIZE;
  cc += GET(cc, 1);
  }

/* No alternative is matched. */

quit = LABEL();

copy_recurse_data(common, ccbegin, ccend, recurse_copy_private_to_global, local_size, private_data_size + local_size, has_quit);

OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(local_size - 1));
free_stack(common, private_data_size + local_size);
OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
OP_SRC(SLJIT_FAST_RETURN, TMP2, 0);

if (common->quit != NULL)
  {
  SLJIT_ASSERT(has_quit);

  set_jumps(common->quit, LABEL());
  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr);
  copy_recurse_data(common, ccbegin, ccend, recurse_copy_shared_to_global, local_size, private_data_size + local_size, has_quit);
  JUMPTO(SLJIT_JUMP, quit);
  }

if (has_accept)
  {
  JUMPHERE(accept_exit);
  free_stack(common, 2);

  /* Save return address. */
  OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(local_size - 1), TMP1, 0);

  copy_recurse_data(common, ccbegin, ccend, recurse_copy_kept_shared_to_global, local_size, private_data_size + local_size, has_quit);

  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(local_size - 1));
  free_stack(common, private_data_size + local_size);
  OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
  OP_SRC(SLJIT_FAST_RETURN, TMP2, 0);
  }

if (common->accept != NULL)
  {
  SLJIT_ASSERT(has_accept);

  set_jumps(common->accept, LABEL());

  OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr);
  OP1(SLJIT_MOV, TMP2, 0, STACK_TOP, 0);

  allocate_stack(common, 2);
  OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, -1);
  }

set_jumps(match, LABEL());

OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0);

copy_recurse_data(common, ccbegin, ccend, recurse_swap_global, local_size, private_data_size + local_size, has_quit);

OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), STACK(local_size - 1));
OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1);
OP_SRC(SLJIT_FAST_RETURN, TMP2, 0);
}


static void check_partial(compiler_common *common, BOOL force)
{
/* Checks whether a partial matching is occurred. Does not modify registers. */
DEFINE_COMPILER;
struct sljit_jump *jump = NULL;

SLJIT_ASSERT(!force || common->mode != PCRE2_JIT_COMPLETE);

if (common->mode == PCRE2_JIT_COMPLETE)
  return;

if (!force && !common->allow_empty_partial)
  jump = CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0);
else if (common->mode == PCRE2_JIT_PARTIAL_SOFT)
  jump = CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, -1);

if (common->mode == PCRE2_JIT_PARTIAL_SOFT)
  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0);
else
  {
  if (common->partialmatchlabel != NULL)
    JUMPTO(SLJIT_JUMP, common->partialmatchlabel);
  else
    add_jump(compiler, &common->partialmatch, JUMP(SLJIT_JUMP));
  }

if (jump != NULL)
  JUMPHERE(jump);
}


static void fast_forward_first_char2(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2, sljit_s32 offset)
{
DEFINE_COMPILER;
struct sljit_label *start;
struct sljit_jump *match;
struct sljit_jump *partial_quit;
PCRE2_UCHAR mask;
BOOL has_match_end = (common->match_end_ptr != 0);

SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE || offset == 0);

if (has_match_end)
  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);

if (offset > 0)
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset));

if (has_match_end)
  {
  OP1(SLJIT_MOV, TMP3, 0, STR_END, 0);

  OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(offset + 1));
  OP2U(SLJIT_SUB | SLJIT_SET_GREATER, STR_END, 0, TMP1, 0);
  CMOV(SLJIT_GREATER, STR_END, TMP1, 0);
  }

#ifdef JIT_HAS_FAST_FORWARD_CHAR_SIMD

if (JIT_HAS_FAST_FORWARD_CHAR_SIMD)
  {
  fast_forward_char_simd(common, char1, char2, offset);

  if (offset > 0)
    OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset));

  if (has_match_end)
    OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
  return;
  }

#endif

start = LABEL();

partial_quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
if (common->mode == PCRE2_JIT_COMPLETE)
  add_jump(compiler, &common->failed_match, partial_quit);

OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));

if (char1 == char2)
  CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, char1, start);
else
  {
  mask = char1 ^ char2;
  if (is_powerof2(mask))
    {
    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, mask);
    CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, char1 | mask, start);
    }
  else
    {
    match = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, char1);
    CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, char2, start);
    JUMPHERE(match);
    }
  }

#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
if (common->utf && offset > 0)
  {
  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-(offset + 1)));
  jumpto_if_not_utf_char_start(compiler, TMP1, start);
  }
#endif

OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset + 1));

if (common->mode != PCRE2_JIT_COMPLETE)
  JUMPHERE(partial_quit);

if (has_match_end)
  OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
}


static void compile_xclass_matchingpath(compiler_common *common, PCRE2_SPTR cc, jump_list **backtracks)
{
DEFINE_COMPILER;
jump_list *found = NULL;
jump_list **list = (cc[0] & XCL_NOT) == 0 ? &found : backtracks;
sljit_uw c, charoffset, max = 256, min = READ_CHAR_MAX;
struct sljit_jump *jump = NULL;
PCRE2_SPTR ccbegin;
int compares, invertcmp, numberofcmps;
#if defined SUPPORT_UNICODE && (PCRE2_CODE_UNIT_WIDTH == 8 || PCRE2_CODE_UNIT_WIDTH == 16)
BOOL utf = common->utf;
#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == [8|16] */

#ifdef SUPPORT_UNICODE
sljit_u32 unicode_status = 0;
int typereg = TMP1;
const sljit_u32 *other_cases;
sljit_uw typeoffset;
#endif /* SUPPORT_UNICODE */

/* Scanning the necessary info. */
cc++;
ccbegin = cc;
compares = 0;

if (cc[-1] & XCL_MAP)
  {
  min = 0;
  cc += 32 / sizeof(PCRE2_UCHAR);
  }

while (*cc != XCL_END)
  {
  compares++;
  if (*cc == XCL_SINGLE)
    {
    cc ++;
    GETCHARINCTEST(c, cc);
    if (c > max) max = c;
    if (c < min) min = c;
#ifdef SUPPORT_UNICODE
    unicode_status |= XCLASS_SAVE_CHAR;
#endif /* SUPPORT_UNICODE */
    }
  else if (*cc == XCL_RANGE)
    {
    cc ++;
    GETCHARINCTEST(c, cc);
    if (c < min) min = c;
    GETCHARINCTEST(c, cc);
    if (c > max) max = c;
#ifdef SUPPORT_UNICODE
    unicode_status |= XCLASS_SAVE_CHAR;
#endif /* SUPPORT_UNICODE */
    }
#ifdef SUPPORT_UNICODE
  else
    {
    SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
    cc++;
    if (*cc == PT_CLIST && cc[-1] == XCL_PROP)
      {
      other_cases = PRIV(ucd_caseless_sets) + cc[1];
      while (*other_cases != NOTACHAR)
        {
        if (*other_cases > max) max = *other_cases;
        if (*other_cases < min) min = *other_cases;
        other_cases++;
        }
      }
    else
      {
      max = READ_CHAR_MAX;
      min = 0;
      }

    switch(*cc)
      {
      case PT_ANY:
      /* Any either accepts everything or ignored. */
      if (cc[-1] == XCL_PROP)
        {
        compile_char1_matchingpath(common, OP_ALLANY, cc, backtracks, FALSE);
        if (list == backtracks)
          add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
        return;
        }
      break;

      case PT_LAMP:
      case PT_GC:
      case PT_PC:
      case PT_ALNUM:
      unicode_status |= XCLASS_HAS_TYPE;
      break;

      case PT_SCX:
      unicode_status |= XCLASS_HAS_SCRIPT_EXTENSION;
      if (cc[-1] == XCL_NOTPROP)
        {
        unicode_status |= XCLASS_SCRIPT_EXTENSION_NOTPROP;
        break;
        }
      compares++;
      /* Fall through */

      case PT_SC:
      unicode_status |= XCLASS_HAS_SCRIPT;
      break;

      case PT_SPACE:
      case PT_PXSPACE:
      case PT_WORD:
      case PT_PXGRAPH:
      case PT_PXPRINT:
      case PT_PXPUNCT:
      unicode_status |= XCLASS_SAVE_CHAR | XCLASS_HAS_TYPE;
      break;

      case PT_CLIST:
      case PT_UCNC:
      unicode_status |= XCLASS_SAVE_CHAR;
      break;

      case PT_BOOL:
      unicode_status |= XCLASS_HAS_BOOL;
      break;

      case PT_BIDICL:
      unicode_status |= XCLASS_HAS_BIDICL;
      break;

      default:
      SLJIT_UNREACHABLE();
      break;
      }
    cc += 2;
    }
#endif /* SUPPORT_UNICODE */
  }
SLJIT_ASSERT(compares > 0);

/* We are not necessary in utf mode even in 8 bit mode. */
cc = ccbegin;
if ((cc[-1] & XCL_NOT) != 0)
  read_char(common, min, max, backtracks, READ_CHAR_UPDATE_STR_PTR);
else
  {
#ifdef SUPPORT_UNICODE
  read_char(common, min, max, (unicode_status & XCLASS_NEEDS_UCD) ? backtracks : NULL, 0);
#else /* !SUPPORT_UNICODE */
  read_char(common, min, max, NULL, 0);
#endif /* SUPPORT_UNICODE */
  }

if ((cc[-1] & XCL_HASPROP) == 0)
  {
  if ((cc[-1] & XCL_MAP) != 0)
    {
    jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255);
    if (!optimize_class(common, (const sljit_u8 *)cc, (((const sljit_u8 *)cc)[31] & 0x80) != 0, TRUE, &found))
      {
      OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
      OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
      OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
      OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
      OP2U(SLJIT_AND | SLJIT_SET_Z, TMP1, 0, TMP2, 0);
      add_jump(compiler, &found, JUMP(SLJIT_NOT_ZERO));
      }

    add_jump(compiler, backtracks, JUMP(SLJIT_JUMP));
    JUMPHERE(jump);

    cc += 32 / sizeof(PCRE2_UCHAR);
    }
  else
    {
    OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, min);
    add_jump(compiler, (cc[-1] & XCL_NOT) == 0 ? backtracks : &found, CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, max - min));
    }
  }
else if ((cc[-1] & XCL_MAP) != 0)
  {
  OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);
#ifdef SUPPORT_UNICODE
  unicode_status |= XCLASS_CHAR_SAVED;
#endif /* SUPPORT_UNICODE */
  if (!optimize_class(common, (const sljit_u8 *)cc, FALSE, TRUE, list))
    {
#if PCRE2_CODE_UNIT_WIDTH == 8
    jump = NULL;
    if (common->utf)
#endif /* PCRE2_CODE_UNIT_WIDTH == 8 */
      jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255);

    OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
    OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
    OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
    OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
    OP2U(SLJIT_AND | SLJIT_SET_Z, TMP1, 0, TMP2, 0);
    add_jump(compiler, list, JUMP(SLJIT_NOT_ZERO));

#if PCRE2_CODE_UNIT_WIDTH == 8
    if (common->utf)
#endif /* PCRE2_CODE_UNIT_WIDTH == 8 */
      JUMPHERE(jump);
    }

  OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0);
  cc += 32 / sizeof(PCRE2_UCHAR);
  }

#ifdef SUPPORT_UNICODE
if (unicode_status & XCLASS_NEEDS_UCD)
  {
  if ((unicode_status & (XCLASS_SAVE_CHAR | XCLASS_CHAR_SAVED)) == XCLASS_SAVE_CHAR)
    OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0);

#if PCRE2_CODE_UNIT_WIDTH == 32
  if (!common->utf)
    {
    jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, MAX_UTF_CODE_POINT + 1);
    OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, UNASSIGNED_UTF_CHAR);
    JUMPHERE(jump);
    }
#endif /* PCRE2_CODE_UNIT_WIDTH == 32 */

  OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
  OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 1);
  OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1));
  OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK);
  OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT);
  OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0);
  OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2));
  OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1);
  OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 3);
  OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 2);
  OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0);

  ccbegin = cc;

  if (unicode_status & XCLASS_HAS_BIDICL)
    {
    OP1(SLJIT_MOV_U16, TMP1, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, scriptx_bidiclass));
    OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BIDICLASS_SHIFT);

    while (*cc != XCL_END)
      {
      if (*cc == XCL_SINGLE)
        {
        cc ++;
        GETCHARINCTEST(c, cc);
        }
      else if (*cc == XCL_RANGE)
        {
        cc ++;
        GETCHARINCTEST(c, cc);
        GETCHARINCTEST(c, cc);
        }
      else
        {
        SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
        cc++;
        if (*cc == PT_BIDICL)
          {
          compares--;
          invertcmp = (compares == 0 && list != backtracks);
          if (cc[-1] == XCL_NOTPROP)
            invertcmp ^= 0x1;
          jump = CMP(SLJIT_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (int)cc[1]);
          add_jump(compiler, compares > 0 ? list : backtracks, jump);
          }
        cc += 2;
        }
      }

    cc = ccbegin;
    }

  if (unicode_status & XCLASS_HAS_BOOL)
    {
    OP1(SLJIT_MOV_U16, TMP1, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, bprops));
    OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BPROPS_MASK);
    OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 2);

    while (*cc != XCL_END)
      {
      if (*cc == XCL_SINGLE)
        {
        cc ++;
        GETCHARINCTEST(c, cc);
        }
      else if (*cc == XCL_RANGE)
        {
        cc ++;
        GETCHARINCTEST(c, cc);
        GETCHARINCTEST(c, cc);
        }
      else
        {
        SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
        cc++;
        if (*cc == PT_BOOL)
          {
          compares--;
          invertcmp = (compares == 0 && list != backtracks);
          if (cc[-1] == XCL_NOTPROP)
            invertcmp ^= 0x1;

          OP2U(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_MEM1(TMP1), (sljit_sw)(PRIV(ucd_boolprop_sets) + (cc[1] >> 5)), SLJIT_IMM, (sljit_sw)1 << (cc[1] & 0x1f));
          add_jump(compiler, compares > 0 ? list : backtracks, JUMP(SLJIT_NOT_ZERO ^ invertcmp));
          }
        cc += 2;
        }
      }

    cc = ccbegin;
    }

  if (unicode_status & XCLASS_HAS_SCRIPT)
    {
    OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, script));

    while (*cc != XCL_END)
      {
      if (*cc == XCL_SINGLE)
        {
        cc ++;
        GETCHARINCTEST(c, cc);
        }
      else if (*cc == XCL_RANGE)
        {
        cc ++;
        GETCHARINCTEST(c, cc);
        GETCHARINCTEST(c, cc);
        }
      else
        {
        SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
        cc++;
        switch (*cc)
          {
          case PT_SCX:
          if (cc[-1] == XCL_NOTPROP)
            break;
          /* Fall through */

          case PT_SC:
          compares--;
          invertcmp = (compares == 0 && list != backtracks);
          if (cc[-1] == XCL_NOTPROP)
            invertcmp ^= 0x1;

          add_jump(compiler, compares > 0 ? list : backtracks, CMP(SLJIT_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (int)cc[1]));
          }
        cc += 2;
        }
      }

    cc = ccbegin;
    }

  if (unicode_status & XCLASS_HAS_SCRIPT_EXTENSION)
    {
    OP1(SLJIT_MOV_U16, TMP1, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, scriptx_bidiclass));
    OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_SCRIPTX_MASK);
    OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 2);

    if (unicode_status & XCLASS_SCRIPT_EXTENSION_NOTPROP)
      {
      if (unicode_status & XCLASS_HAS_TYPE)
        {
        if (unicode_status & XCLASS_SAVE_CHAR)
          {
          OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, TMP2, 0);
          unicode_status |= XCLASS_SCRIPT_EXTENSION_RESTORE_LOCALS0;
          }
        else
          {
          OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP2, 0);
          unicode_status |= XCLASS_SCRIPT_EXTENSION_RESTORE_RETURN_ADDR;
          }
        }
      OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, script));
      }

    while (*cc != XCL_END)
      {
      if (*cc == XCL_SINGLE)
        {
        cc ++;
        GETCHARINCTEST(c, cc);
        }
      else if (*cc == XCL_RANGE)
        {
        cc ++;
        GETCHARINCTEST(c, cc);
        GETCHARINCTEST(c, cc);
        }
      else
        {
        SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
        cc++;
        if (*cc == PT_SCX)
          {
          compares--;
          invertcmp = (compares == 0 && list != backtracks);

          jump = NULL;
          if (cc[-1] == XCL_NOTPROP)
            {
            jump = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, (int)cc[1]);
            if (invertcmp)
              {
              add_jump(compiler, backtracks, jump);
              jump = NULL;
              }
            invertcmp ^= 0x1;
            }

          OP2U(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_MEM1(TMP1), (sljit_sw)(PRIV(ucd_script_sets) + (cc[1] >> 5)), SLJIT_IMM, (sljit_sw)1 << (cc[1] & 0x1f));
          add_jump(compiler, compares > 0 ? list : backtracks, JUMP(SLJIT_NOT_ZERO ^ invertcmp));

          if (jump != NULL)
            JUMPHERE(jump);
          }
        cc += 2;
        }
      }

    if (unicode_status & XCLASS_SCRIPT_EXTENSION_RESTORE_LOCALS0)
      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
    else if (unicode_status & XCLASS_SCRIPT_EXTENSION_RESTORE_RETURN_ADDR)
      OP1(SLJIT_MOV, TMP2, 0, RETURN_ADDR, 0);
    cc = ccbegin;
    }

  if (unicode_status & XCLASS_SAVE_CHAR)
    OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0);

  if (unicode_status & XCLASS_HAS_TYPE)
    {
    if (unicode_status & XCLASS_SAVE_CHAR)
      typereg = RETURN_ADDR;

    OP1(SLJIT_MOV_U8, typereg, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype));
    }
  }
#endif /* SUPPORT_UNICODE */

/* Generating code. */
charoffset = 0;
numberofcmps = 0;
#ifdef SUPPORT_UNICODE
typeoffset = 0;
#endif /* SUPPORT_UNICODE */

while (*cc != XCL_END)
  {
  compares--;
  invertcmp = (compares == 0 && list != backtracks);
  jump = NULL;

  if (*cc == XCL_SINGLE)
    {
    cc ++;
    GETCHARINCTEST(c, cc);

    if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
      {
      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
      OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
      numberofcmps++;
      }
    else if (numberofcmps > 0)
      {
      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL);
      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
      numberofcmps = 0;
      }
    else
      {
      jump = CMP(SLJIT_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
      numberofcmps = 0;
      }
    }
  else if (*cc == XCL_RANGE)
    {
    cc ++;
    GETCHARINCTEST(c, cc);
    SET_CHAR_OFFSET(c);
    GETCHARINCTEST(c, cc);

    if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE))
      {
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
      OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL);
      numberofcmps++;
      }
    else if (numberofcmps > 0)
      {
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL);
      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
      numberofcmps = 0;
      }
    else
      {
      jump = CMP(SLJIT_LESS_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset));
      numberofcmps = 0;
      }
    }
#ifdef SUPPORT_UNICODE
  else
    {
    SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP);
    if (*cc == XCL_NOTPROP)
      invertcmp ^= 0x1;
    cc++;
    switch(*cc)
      {
      case PT_ANY:
      if (!invertcmp)
        jump = JUMP(SLJIT_JUMP);
      break;

      case PT_LAMP:
      OP2U(SLJIT_SUB | SLJIT_SET_Z, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset);
      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
      OP2U(SLJIT_SUB | SLJIT_SET_Z, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset);
      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
      OP2U(SLJIT_SUB | SLJIT_SET_Z, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset);
      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL);
      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
      break;

      case PT_GC:
      c = PRIV(ucp_typerange)[(int)cc[1] * 2];
      SET_TYPE_OFFSET(c);
      jump = CMP(SLJIT_LESS_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, PRIV(ucp_typerange)[(int)cc[1] * 2 + 1] - c);
      break;

      case PT_PC:
      jump = CMP(SLJIT_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, (int)cc[1] - typeoffset);
      break;

      case PT_SC:
      case PT_SCX:
      case PT_BOOL:
      case PT_BIDICL:
      compares++;
      /* Do nothing. */
      break;

      case PT_SPACE:
      case PT_PXSPACE:
      SET_CHAR_OFFSET(9);
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, TMP1, 0, SLJIT_IMM, 0xd - 0x9);
      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);

      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, 0x85 - 0x9);
      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);

      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, 0x180e - 0x9);
      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);

      SET_TYPE_OFFSET(ucp_Zl);
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl);
      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL);
      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
      break;

      case PT_WORD:
      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_UNDERSCORE - charoffset));
      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
      /* Fall through. */

      case PT_ALNUM:
      SET_TYPE_OFFSET(ucp_Ll);
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll);
      OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL);
      SET_TYPE_OFFSET(ucp_Nd);
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd);
      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL);
      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
      break;

      case PT_CLIST:
      other_cases = PRIV(ucd_caseless_sets) + cc[1];

      /* At least three characters are required.
         Otherwise this case would be handled by the normal code path. */
      SLJIT_ASSERT(other_cases[0] != NOTACHAR && other_cases[1] != NOTACHAR && other_cases[2] != NOTACHAR);
      SLJIT_ASSERT(other_cases[0] < other_cases[1] && other_cases[1] < other_cases[2]);

      /* Optimizing character pairs, if their difference is power of 2. */
      if (is_powerof2(other_cases[1] ^ other_cases[0]))
        {
        if (charoffset == 0)
          OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]);
        else
          {
          OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset);
          OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]);
          }
        OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_IMM, other_cases[1]);
        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
        other_cases += 2;
        }
      else if (is_powerof2(other_cases[2] ^ other_cases[1]))
        {
        if (charoffset == 0)
          OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, other_cases[2] ^ other_cases[1]);
        else
          {
          OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset);
          OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]);
          }
        OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_IMM, other_cases[2]);
        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);

        OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, (sljit_sw)(other_cases[0] - charoffset));
        OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_Z : 0), TMP2, 0, SLJIT_EQUAL);

        other_cases += 3;
        }
      else
        {
        OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset));
        OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
        }

      while (*other_cases != NOTACHAR)
        {
        OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset));
        OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_Z : 0), TMP2, 0, SLJIT_EQUAL);
        }
      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
      break;

      case PT_UCNC:
      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_DOLLAR_SIGN - charoffset));
      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL);
      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_COMMERCIAL_AT - charoffset));
      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);
      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_GRAVE_ACCENT - charoffset));
      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);

      SET_CHAR_OFFSET(0xa0);
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, TMP1, 0, SLJIT_IMM, (sljit_sw)(0xd7ff - charoffset));
      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL);
      SET_CHAR_OFFSET(0);
      OP2U(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0xe000 - 0);
      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_GREATER_EQUAL);
      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
      break;

      case PT_PXGRAPH:
      /* C and Z groups are the farthest two groups. */
      SET_TYPE_OFFSET(ucp_Ll);
      OP2U(SLJIT_SUB | SLJIT_SET_GREATER, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll);
      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER);

      jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll);

      /* In case of ucp_Cf, we overwrite the result. */
      SET_CHAR_OFFSET(0x2066);
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066);
      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);

      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066);
      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);

      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, 0x180e - 0x2066);
      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);

      JUMPHERE(jump);
      jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0);
      break;

      case PT_PXPRINT:
      /* C and Z groups are the farthest two groups. */
      SET_TYPE_OFFSET(ucp_Ll);
      OP2U(SLJIT_SUB | SLJIT_SET_GREATER, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll);
      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER);

      OP2U(SLJIT_SUB | SLJIT_SET_Z, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Ll);
      OP_FLAGS(SLJIT_AND, TMP2, 0, SLJIT_NOT_EQUAL);

      jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll);

      /* In case of ucp_Cf, we overwrite the result. */
      SET_CHAR_OFFSET(0x2066);
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066);
      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);

      OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066);
      OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL);

      JUMPHERE(jump);
      jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0);
      break;

      case PT_PXPUNCT:
      SET_TYPE_OFFSET(ucp_Sc);
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, typereg, 0, SLJIT_IMM, ucp_So - ucp_Sc);
      OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL);

      SET_CHAR_OFFSET(0);
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, TMP1, 0, SLJIT_IMM, 0x7f);
      OP_FLAGS(SLJIT_AND, TMP2, 0, SLJIT_LESS_EQUAL);

      SET_TYPE_OFFSET(ucp_Pc);
      OP2U(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, typereg, 0, SLJIT_IMM, ucp_Ps - ucp_Pc);
      OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL);
      jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp);
      break;

      default:
      SLJIT_UNREACHABLE();
      break;
      }
    cc += 2;
    }
#endif /* SUPPORT_UNICODE */

  if (jump != NULL)
    add_jump(compiler, compares > 0 ? list : backtracks, jump);
  }

if (found != NULL)
  set_jumps(found, LABEL());
}


static SLJIT_INLINE BOOL char_has_othercase(compiler_common *common, PCRE2_SPTR cc)
{
/* Detects if the character has an othercase. */
unsigned int c;

#ifdef SUPPORT_UNICODE
if (common->utf || common->ucp)
  {
  if (common->utf)
    {
    GETCHAR(c, cc);
    }
  else
    c = *cc;

  if (c > 127)
    return c != UCD_OTHERCASE(c);

  return common->fcc[c] != c;
  }
else
#endif
  c = *cc;
return MAX_255(c) ? common->fcc[c] != c : FALSE;
}


static SLJIT_INLINE PCRE2_SPTR compile_callout_matchingpath(compiler_common *common, PCRE2_SPTR cc, backtrack_common *parent)
{
DEFINE_COMPILER;
backtrack_common *backtrack;
sljit_s32 mov_opcode;
unsigned int callout_length = (*cc == OP_CALLOUT)
    ? PRIV(OP_lengths)[OP_CALLOUT] : GET(cc, 1 + 2 * LINK_SIZE);
sljit_sw value1;
sljit_sw value2;
sljit_sw value3;
sljit_uw callout_arg_size = (common->re->top_bracket + 1) * 2 * sizeof(sljit_sw);

PUSH_BACKTRACK(sizeof(backtrack_common), cc, NULL);

callout_arg_size = (sizeof(pcre2_callout_block) + callout_arg_size + sizeof(sljit_sw) - 1) / sizeof(sljit_sw);

allocate_stack(common, callout_arg_size);

SLJIT_ASSERT(common->capture_last_ptr != 0);
OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr);
OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
value1 = (*cc == OP_CALLOUT) ? cc[1 + 2 * LINK_SIZE] : 0;
OP1(SLJIT_MOV_U32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(callout_number), SLJIT_IMM, value1);
OP1(SLJIT_MOV_U32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(capture_last), TMP2, 0);
OP1(SLJIT_MOV_U32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(capture_top), SLJIT_IMM, common->re->top_bracket + 1);

/* These pointer sized fields temporarly stores internal variables. */
OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(offset_vector), STR_PTR, 0);

if (common->mark_ptr != 0)
  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr));
mov_opcode = (sizeof(PCRE2_SIZE) == 4) ? SLJIT_MOV_U32 : SLJIT_MOV;
OP1(mov_opcode, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(pattern_position), SLJIT_IMM, GET(cc, 1));
OP1(mov_opcode, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(next_item_length), SLJIT_IMM, GET(cc, 1 + LINK_SIZE));

if (*cc == OP_CALLOUT)
  {
  value1 = 0;
  value2 = 0;
  value3 = 0;
  }
else
  {
  value1 = (sljit_sw) (cc + (1 + 4*LINK_SIZE) + 1);
  value2 = (callout_length - (1 + 4*LINK_SIZE + 2));
  value3 = (sljit_sw) (GET(cc, 1 + 3*LINK_SIZE));
  }

OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(callout_string), SLJIT_IMM, value1);
OP1(mov_opcode, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(callout_string_length), SLJIT_IMM, value2);
OP1(mov_opcode, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(callout_string_offset), SLJIT_IMM, value3);
OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(mark), (common->mark_ptr != 0) ? TMP2 : SLJIT_IMM, 0);

SLJIT_ASSERT(TMP1 == SLJIT_R0 && STR_PTR == SLJIT_R1);

/* Needed to save important temporary registers. */
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STR_PTR, 0);
/* SLJIT_R0 = arguments */
OP1(SLJIT_MOV, SLJIT_R1, 0, STACK_TOP, 0);
GET_LOCAL_BASE(SLJIT_R2, 0, OVECTOR_START);
sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS3(32, W, W, W), SLJIT_IMM, SLJIT_FUNC_ADDR(do_callout));
OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
free_stack(common, callout_arg_size);

/* Check return value. */
OP2U(SLJIT_SUB32 | SLJIT_SET_Z | SLJIT_SET_SIG_GREATER, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_SIG_GREATER));
if (common->abort_label == NULL)
  add_jump(compiler, &common->abort, JUMP(SLJIT_NOT_EQUAL) /* SIG_LESS */);
else
  JUMPTO(SLJIT_NOT_EQUAL /* SIG_LESS */, common->abort_label);
return cc + callout_length;
}


static sljit_s32 SLJIT_FUNC do_callout(struct jit_arguments *arguments, pcre2_callout_block *callout_block, PCRE2_SPTR *jit_ovector)
{
PCRE2_SPTR begin;
PCRE2_SIZE *ovector;
sljit_u32 oveccount, capture_top;

if (arguments->callout == NULL)
  return 0;

SLJIT_COMPILE_ASSERT(sizeof (PCRE2_SIZE) <= sizeof (sljit_sw), pcre2_size_must_be_lower_than_sljit_sw_size);

begin = arguments->begin;
ovector = (PCRE2_SIZE*)(callout_block + 1);
oveccount = callout_block->capture_top;

SLJIT_ASSERT(oveccount >= 1);

callout_block->version = 2;
callout_block->callout_flags = 0;

/* Offsets in subject. */
callout_block->subject_length = arguments->end - arguments->begin;
callout_block->start_match = jit_ovector[0] - begin;
callout_block->current_position = (PCRE2_SPTR)callout_block->offset_vector - begin;
callout_block->subject = begin;

/* Convert and copy the JIT offset vector to the ovector array. */
callout_block->capture_top = 1;
callout_block->offset_vector = ovector;

ovector[0] = PCRE2_UNSET;
ovector[1] = PCRE2_UNSET;
ovector += 2;
jit_ovector += 2;
capture_top = 1;

/* Convert pointers to sizes. */
while (--oveccount != 0)
  {
  capture_top++;

  ovector[0] = (PCRE2_SIZE)(jit_ovector[0] - begin);
  ovector[1] = (PCRE2_SIZE)(jit_ovector[1] - begin);

  if (ovector[0] != PCRE2_UNSET)
    callout_block->capture_top = capture_top;

  ovector += 2;
  jit_ovector += 2;
  }

return (arguments->callout)(callout_block, arguments->callout_data);
}


static SLJIT_INLINE void add_jump(struct sljit_compiler *compiler, jump_list **list, struct sljit_jump *jump)
{
jump_list *list_item = sljit_alloc_memory(compiler, sizeof(jump_list));
if (list_item)
  {
  list_item->next = *list;
  list_item->jump = jump;
  *list = list_item;
  }
}


static SLJIT_INLINE void compile_ref_iterator_backtrackingpath(compiler_common *common, struct backtrack_common *current)
{
DEFINE_COMPILER;
PCRE2_SPTR cc = current->cc;
BOOL ref = (*cc == OP_REF || *cc == OP_REFI);
PCRE2_UCHAR type;

type = cc[ref ? 1 + IMM2_SIZE : 1 + 2 * IMM2_SIZE];

if ((type & 0x1) == 0)
  {
  /* Maximize case. */
  set_jumps(current->topbacktracks, LABEL());
  OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
  free_stack(common, 1);
  CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(ref_iterator_backtrack)->matchingpath);
  return;
  }

OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(ref_iterator_backtrack)->matchingpath);
set_jumps(current->topbacktracks, LABEL());
free_stack(common, ref ? 2 : 3);
}


static SLJIT_INLINE jump_list *search_requested_char(compiler_common *common, PCRE2_UCHAR req_char, BOOL caseless, BOOL has_firstchar)
{
DEFINE_COMPILER;
struct sljit_label *loop;
struct sljit_jump *toolong;
struct sljit_jump *already_found;
struct sljit_jump *found;
struct sljit_jump *found_oc = NULL;
jump_list *not_found = NULL;
sljit_u32 oc, bit;

SLJIT_ASSERT(common->req_char_ptr != 0);
OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(REQ_CU_MAX) * 100);
OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr);
toolong = CMP(SLJIT_LESS, TMP2, 0, STR_END, 0);
already_found = CMP(SLJIT_LESS, STR_PTR, 0, TMP1, 0);

if (has_firstchar)
  OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
else
  OP1(SLJIT_MOV, TMP1, 0, STR_PTR, 0);

oc = req_char;
if (caseless)
  {
  oc = TABLE_GET(req_char, common->fcc, req_char);
#if defined SUPPORT_UNICODE
  if (req_char > 127 && (common->utf || common->ucp))
    oc = UCD_OTHERCASE(req_char);
#endif
  }

#ifdef JIT_HAS_FAST_REQUESTED_CHAR_SIMD
if (JIT_HAS_FAST_REQUESTED_CHAR_SIMD)
  {
  not_found = fast_requested_char_simd(common, req_char, oc);
  }
else
#endif
  {
  loop = LABEL();
  add_jump(compiler, &not_found, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0));

  OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(TMP1), 0);

  if (req_char == oc)
    found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char);
  else
    {
    bit = req_char ^ oc;
    if (is_powerof2(bit))
      {
       OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, bit);
      found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char | bit);
      }
    else
      {
      found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char);
      found_oc = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, oc);
      }
    }
  OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
  JUMPTO(SLJIT_JUMP, loop);

  JUMPHERE(found);
  if (found_oc)
    JUMPHERE(found_oc);
  }

OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr, TMP1, 0);

JUMPHERE(already_found);
JUMPHERE(toolong);
return not_found;
}


static void read_char8_type(compiler_common *common, jump_list **backtracks, BOOL negated)
{
/* Reads the character type into TMP1, updates STR_PTR. Does not check STR_END. */
DEFINE_COMPILER;
#if defined SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH != 8
struct sljit_jump *jump;
#endif
#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8
struct sljit_jump *jump2;
#endif

SLJIT_UNUSED_ARG(backtracks);
SLJIT_UNUSED_ARG(negated);

OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), 0);
OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));

#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8
if (common->utf)
  {
  /* The result of this read may be unused, but saves an "else" part. */
  OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
  jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0x80);

  if (!negated)
    {
    if (common->invalid_utf)
      add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));

    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
    OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc2);
    if (common->invalid_utf)
      add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0xe0 - 0xc2));

    OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
    OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0);
    OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x80);
    if (common->invalid_utf)
      add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40));

    OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
    jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255);
    OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
    JUMPHERE(jump2);
    }
  else if (common->invalid_utf)
    {
    add_jump(compiler, &common->utfreadchar_invalid, JUMP(SLJIT_FAST_CALL));
    OP1(SLJIT_MOV, TMP2, 0, TMP1, 0);
    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR));

    OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
    jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255);
    OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
    JUMPHERE(jump2);
    }
  else
    add_jump(compiler, &common->utfreadtype8, JUMP(SLJIT_FAST_CALL));

  JUMPHERE(jump);
  return;
  }
#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 */

#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 32
if (common->invalid_utf && negated)
  add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x110000));
#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 32 */

#if PCRE2_CODE_UNIT_WIDTH != 8
/* The ctypes array contains only 256 values. */
OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255);
#endif /* PCRE2_CODE_UNIT_WIDTH != 8 */
OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes);
#if PCRE2_CODE_UNIT_WIDTH != 8
JUMPHERE(jump);
#endif /* PCRE2_CODE_UNIT_WIDTH != 8 */

#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 16
if (common->utf && negated)
  {
  /* Skip low surrogate if necessary. */
  if (!common->invalid_utf)
    {
    OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xd800);

    if (sljit_has_cpu_feature(SLJIT_HAS_CMOV) && !HAS_VIRTUAL_REGISTERS)
      {
      OP2(SLJIT_ADD, RETURN_ADDR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
      OP2U(SLJIT_SUB | SLJIT_SET_LESS, TMP2, 0, SLJIT_IMM, 0x400);
      CMOV(SLJIT_LESS, STR_PTR, RETURN_ADDR, 0);
      }
    else
      {
      jump = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x400);
      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
      JUMPHERE(jump);
      }
    return;
    }

  OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xd800);
  jump = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0xe000 - 0xd800);
  add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x400));
  add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));

  OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xdc00);
  add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x400));

  JUMPHERE(jump);
  return;
  }
#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 16 */
}


static void do_utfpeakcharback(compiler_common *common)
{
/* Peak a character back. Does not modify STR_PTR. */
DEFINE_COMPILER;
struct sljit_jump *jump[2];

sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);

OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2));
OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0);
jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x20);

OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-3));
OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0);
jump[1] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10);

OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-4));
OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0 - 0x80);
OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf0);
OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6);
OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);

JUMPHERE(jump[1]);
OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2));
OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x80);
OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);

JUMPHERE(jump[0]);
OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1));
OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6);
OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x80);
OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0);

OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0);
}


static void compile_dnref_search(compiler_common *common, PCRE2_SPTR cc, jump_list **backtracks)
{
/* The OVECTOR offset goes to TMP2. */
DEFINE_COMPILER;
int count = GET2(cc, 1 + IMM2_SIZE);
PCRE2_SPTR slot = common->name_table + GET2(cc, 1) * common->name_entry_size;
unsigned int offset;
jump_list *found = NULL;

SLJIT_ASSERT(*cc == OP_DNREF || *cc == OP_DNREFI);

OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1));

count--;
while (count-- > 0)
  {
  offset = GET2(slot, 0) << 1;
  GET_LOCAL_BASE(TMP2, 0, OVECTOR(offset));
  add_jump(compiler, &found, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0));
  slot += common->name_entry_size;
  }

offset = GET2(slot, 0) << 1;
GET_LOCAL_BASE(TMP2, 0, OVECTOR(offset));
if (backtracks != NULL && !common->unset_backref)
  add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0));

set_jumps(found, LABEL());
}


static void compile_iterator_backtrackingpath(compiler_common *common, struct backtrack_common *current)
{
DEFINE_COMPILER;
PCRE2_SPTR cc = current->cc;
PCRE2_UCHAR opcode;
PCRE2_UCHAR type;
sljit_u32 max = 0, exact;
struct sljit_label *label = NULL;
struct sljit_jump *jump = NULL;
jump_list *jumplist = NULL;
PCRE2_SPTR end;
int private_data_ptr = PRIVATE_DATA(cc);
int base = (private_data_ptr == 0) ? SLJIT_MEM1(STACK_TOP) : SLJIT_MEM1(SLJIT_SP);
int offset0 = (private_data_ptr == 0) ? STACK(0) : private_data_ptr;
int offset1 = (private_data_ptr == 0) ? STACK(1) : private_data_ptr + (int)sizeof(sljit_sw);

cc = get_iterator_parameters(common, cc, &opcode, &type, &max, &exact, &end);

switch(opcode)
  {
  case OP_STAR:
  case OP_UPTO:
  if (type == OP_ANYNL || type == OP_EXTUNI)
    {
    SLJIT_ASSERT(private_data_ptr == 0);
    set_jumps(CURRENT_AS(char_iterator_backtrack)->u.backtracks, LABEL());
    OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0));
    free_stack(common, 1);
    CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(char_iterator_backtrack)->matchingpath);
    }
  else
    {
    if (CURRENT_AS(char_iterator_backtrack)->u.charpos.enabled)
      {
      OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
      OP1(SLJIT_MOV, TMP2, 0, base, offset1);
      OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));

      jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0);
      label = LABEL();
      OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1));
      OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
      if (CURRENT_AS(char_iterator_backtrack)->u.charpos.othercasebit != 0)
        OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, CURRENT_AS(char_iterator_backtrack)->u.charpos.othercasebit);
      CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CURRENT_AS(char_iterator_backtrack)->u.charpos.chr, CURRENT_AS(char_iterator_backtrack)->matchingpath);
      move_back(common, NULL, TRUE);
      CMPTO(SLJIT_GREATER, STR_PTR, 0, TMP2, 0, label);
      }
    else
      {
      OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
      jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, base, offset1);
      move_back(common, NULL, TRUE);
      OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
      JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath);
      }
    JUMPHERE(jump);
    if (private_data_ptr == 0)
      free_stack(common, 2);
    }
  break;

  case OP_MINSTAR:
  OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
  compile_char1_matchingpath(common, type, cc, &jumplist, TRUE);
  OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
  JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath);
  set_jumps(jumplist, LABEL());
  if (private_data_ptr == 0)
    free_stack(common, 1);
  break;

  case OP_MINUPTO:
  OP1(SLJIT_MOV, TMP1, 0, base, offset1);
  OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
  OP2(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
  add_jump(compiler, &jumplist, JUMP(SLJIT_ZERO));

  OP1(SLJIT_MOV, base, offset1, TMP1, 0);
  compile_char1_matchingpath(common, type, cc, &jumplist, TRUE);
  OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
  JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath);

  set_jumps(jumplist, LABEL());
  if (private_data_ptr == 0)
    free_stack(common, 2);
  break;

  case OP_QUERY:
  OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
  OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0);
  CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(char_iterator_backtrack)->matchingpath);
  jump = JUMP(SLJIT_JUMP);
  set_jumps(CURRENT_AS(char_iterator_backtrack)->u.backtracks, LABEL());
  OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
  OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0);
  JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath);
  JUMPHERE(jump);
  if (private_data_ptr == 0)
    free_stack(common, 1);
  break;

  case OP_MINQUERY:
  OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
  OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0);
  jump = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0);
  compile_char1_matchingpath(common, type, cc, &jumplist, TRUE);
  JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath);
  set_jumps(jumplist, LABEL());
  JUMPHERE(jump);
  if (private_data_ptr == 0)
    free_stack(common, 1);
  break;

  case OP_EXACT:
  case OP_POSSTAR:
  case OP_POSQUERY:
  case OP_POSUPTO:
  break;

  default:
  SLJIT_UNREACHABLE();
  break;
  }

set_jumps(current->topbacktracks, LABEL());
}


static void do_casefulcmp(compiler_common *common)
{
DEFINE_COMPILER;
struct sljit_jump *jump;
struct sljit_label *label;
int char1_reg;
int char2_reg;

if (HAS_VIRTUAL_REGISTERS)
  {
  char1_reg = STR_END;
  char2_reg = STACK_TOP;
  }
else
  {
  char1_reg = TMP3;
  char2_reg = RETURN_ADDR;
  }

sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0);
OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);

if (char1_reg == STR_END)
  {
  OP1(SLJIT_MOV, TMP3, 0, char1_reg, 0);
  OP1(SLJIT_MOV, RETURN_ADDR, 0, char2_reg, 0);
  }

if (sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_SUPP | SLJIT_MEM_POST, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1)) == SLJIT_SUCCESS)
  {
  label = LABEL();
  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_POST, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1));
  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_POST, char2_reg, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
  jump = CMP(SLJIT_NOT_EQUAL, char1_reg, 0, char2_reg, 0);
  OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
  JUMPTO(SLJIT_NOT_ZERO, label);

  JUMPHERE(jump);
  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
  }
else if (sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_SUPP | SLJIT_MEM_PRE, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1)) == SLJIT_SUCCESS)
  {
  OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));

  label = LABEL();
  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_PRE, char1_reg, SLJIT_MEM1(TMP1), IN_UCHARS(1));
  sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_PRE, char2_reg, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));
  jump = CMP(SLJIT_NOT_EQUAL, char1_reg, 0, char2_reg, 0);
  OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
  JUMPTO(SLJIT_NOT_ZERO, label);

  JUMPHERE(jump);
  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  }
else
  {
  label = LABEL();
  OP1(MOV_UCHAR, char1_reg, 0, SLJIT_MEM1(TMP1), 0);
  OP1(MOV_UCHAR, char2_reg, 0, SLJIT_MEM1(STR_PTR), 0);
  OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1));
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  jump = CMP(SLJIT_NOT_EQUAL, char1_reg, 0, char2_reg, 0);
  OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1));
  JUMPTO(SLJIT_NOT_ZERO, label);

  JUMPHERE(jump);
  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
  }

if (char1_reg == STR_END)
  {
  OP1(SLJIT_MOV, char1_reg, 0, TMP3, 0);
  OP1(SLJIT_MOV, char2_reg, 0, RETURN_ADDR, 0);
  }

OP_SRC(SLJIT_FAST_RETURN, TMP1, 0);
}