void AutoParallel::BuildGraph(GraphDef* graph) {
  AddSharedNodes(graph);
  for (int i = 0; i < num_replicas_; i++) {
    AddOneReplica(graph, i);
  }
  std::set<string> fetches;
  for (size_t i = 0; i < item_->fetch.size(); i++) {
    for (int j = 0; j < num_replicas_; j++) {
      string prefix = strings::StrCat(kAutoParallelPrefix, "-Replica-", j);
      string fetch = AddPrefixToNodeName(item_->fetch[i], prefix);
      fetches.insert(fetch);
    }
  }
  string name_control =
      strings::StrCat(kAutoParallelPrefix, "-Control-", "Fetch");
  auto control = AddNodeControl(name_control, fetches, graph);

  for (const auto& fetch : item_->fetch) {
    AddNodeControl(fetch, {control->name()}, graph);
  }
  *graph->mutable_library() = item_->graph.library();
  *graph->mutable_versions() = item_->graph.versions();
  LOG(INFO) << "Parallelized graph size: " << graph->node_size();
}


Prevent unitialized variable use in grappler.

PiperOrigin-RevId: 399702928
Change-Id: Id7e75451fbff297692dfb687f60ea04b25c96b24

TEST_F(AutoParallelTest, SimpleParallel) {
  tensorflow::Scope s = tensorflow::Scope::DisabledShapeInferenceScope();
  Output constant_a = ops::Const(s.WithOpName("constant_a"), 1.0f, {1});
  Output constant_b = ops::Const(s.WithOpName("constant_b"), 1, {1});
  Output var = ops::Variable(s.WithOpName("var"), {1}, DT_FLOAT);
  Output assign = ops::Assign(s.WithOpName("assign"), {var}, {constant_a});
  Output identity = ops::Identity(s.WithOpName("identity"), {var});
  Output fifo_queue = ops::FIFOQueue(s.WithOpName("fifo_queue"), {DT_FLOAT});
  auto dequeue = ops::QueueDequeueMany(s.WithOpName("dequeue"), {fifo_queue},
                                       {constant_b}, {DT_FLOAT});
  Output add = ops::AddN(s.WithOpName("add"), {constant_a, dequeue[0]});
  Output learning_rate = ops::Const(s.WithOpName("learning_rate"), 0.01f, {1});
  Output apply_gradient = ops::ApplyGradientDescent(
      s.WithOpName("apply_gradient"), {var}, {learning_rate}, {add});

  GrapplerItem item;
  item.init_ops.push_back("assign");
  item.fetch.push_back("apply_gradient");
  item.init_ops.push_back("assign");
  TF_CHECK_OK(s.ToGraphDef(&item.graph));

  AutoParallel parallel(2);
  GraphDef output;
  Status status = parallel.Optimize(nullptr, item, &output);
  TF_EXPECT_OK(status);
  EXPECT_EQ(21, output.node_size());

  const NodeDef& node_assign = output.node(0);
  EXPECT_EQ("assign", node_assign.name());
  EXPECT_EQ("AutoParallel-Replica-0/constant_a", node_assign.input(1));

  const NodeDef& node_constant_b = output.node(1);
  EXPECT_EQ("constant_b", node_constant_b.name());

  const NodeDef& node_fifo_queue = output.node(2);
  EXPECT_EQ("fifo_queue", node_fifo_queue.name());

  const NodeDef& node_identity = output.node(3);
  EXPECT_EQ("identity", node_identity.name());
  EXPECT_EQ("var", node_identity.input(0));

  const NodeDef& node_var = output.node(4);
  EXPECT_EQ("var", node_var.name());

  const NodeDef& node_div_const0 = output.node(5);
  EXPECT_EQ("AutoParallel-Replica-0/AutoParallel-Div-Const",
            node_div_const0.name());

  const NodeDef& node_div0 = output.node(6);
  EXPECT_EQ("AutoParallel-Replica-0/AutoParallel-Div-apply_gradient",
            node_div0.name());
  const NodeDef& node_add0 = output.node(7);
  EXPECT_EQ("AutoParallel-Replica-0/add", node_add0.name());

  const NodeDef& node_gradient0 = output.node(8);
  EXPECT_EQ("AutoParallel-Replica-0/apply_gradient", node_gradient0.name());

  const NodeDef& node_constant_a0 = output.node(9);
  EXPECT_EQ("AutoParallel-Replica-0/constant_a", node_constant_a0.name());

  const NodeDef& node_dequeue0 = output.node(10);
  EXPECT_EQ("AutoParallel-Replica-0/dequeue", node_dequeue0.name());

  const NodeDef& node_learning_rate0 = output.node(11);
  EXPECT_EQ("AutoParallel-Replica-0/learning_rate", node_learning_rate0.name());

  const NodeDef& node_div_const1 = output.node(12);
  EXPECT_EQ("AutoParallel-Replica-1/AutoParallel-Div-Const",
            node_div_const1.name());

  const NodeDef& node_div1 = output.node(13);
  EXPECT_EQ("AutoParallel-Replica-1/AutoParallel-Div-apply_gradient",
            node_div1.name());

  const NodeDef& node_add1 = output.node(14);
  EXPECT_EQ("AutoParallel-Replica-1/add", node_add1.name());

  const NodeDef& node_gradient1 = output.node(15);
  EXPECT_EQ("AutoParallel-Replica-1/apply_gradient", node_gradient1.name());

  const NodeDef& node_constant_a1 = output.node(16);
  EXPECT_EQ("AutoParallel-Replica-1/constant_a", node_constant_a1.name());

  const NodeDef& node_dequeue1 = output.node(17);
  EXPECT_EQ("AutoParallel-Replica-1/dequeue", node_dequeue1.name());

  const NodeDef& node_learning_rate1 = output.node(18);
  EXPECT_EQ("AutoParallel-Replica-1/learning_rate", node_learning_rate1.name());

  const NodeDef& node_fetch = output.node(19);
  EXPECT_EQ("AutoParallel-Control-Fetch", node_fetch.name());
  EXPECT_EQ("^AutoParallel-Replica-0/apply_gradient", node_fetch.input(0));
  EXPECT_EQ("^AutoParallel-Replica-1/apply_gradient", node_fetch.input(1));

  const NodeDef& node_gradient = output.node(20);
  EXPECT_EQ("apply_gradient", node_gradient.name());
  EXPECT_EQ("^AutoParallel-Control-Fetch", node_gradient.input(0));
}

TensorFlow is an open source platform for machine learning. In affected versions TensorFlow's Grappler optimizer has a use of unitialized variable. If the `train_nodes` vector (obtained from the saved model that gets optimized) does not contain a `Dequeue` node, then `dequeue_node` is left unitialized. The fix will be included in TensorFlow 2.7.0. We will also cherrypick this commit on TensorFlow 2.6.1, TensorFlow 2.5.2, and TensorFlow 2.4.4, as these are also affected and still in supported range.


Status AutoParallel::Optimize(Cluster* cluster, const GrapplerItem& item,
                              GraphDef* output) {
  TF_RETURN_IF_ERROR(Initialize(item));
  BuildGraph(output);
  return Status::OK();
}


NodeDef* AutoParallel::AddNodeDivConst() {
  NodeDef* node = graph_.add_node();
  node->set_name(strings::StrCat(kAutoParallelPrefix, "-Div-Const"));
  node->set_op("Const");

  AttrValue attr_data_type;
  attr_data_type.set_type(DT_FLOAT);
  node->mutable_attr()->insert({"dtype", attr_data_type});

  AttrValue attr_tensor;
  auto tensor = attr_tensor.mutable_tensor();
  tensor->add_float_val(static_cast<float>(num_replicas_));
  tensor->set_dtype(DT_FLOAT);
  node->mutable_attr()->insert({"value", attr_tensor});
  return node;
}


NodeDef* AutoParallel::AddNodeControl(const string& name,
                                      const std::set<string>& deps,
                                      GraphDef* graph) {
  NodeDef* node = graph->add_node();
  node->set_name(name);
  node->set_op("NoOp");
  for (const auto& dep : deps) {
    node->add_input(strings::StrCat("^", dep));
  }
  return node;
}


Status AutoParallel::Initialize(const GrapplerItem& item) {
  num_gpus_ = GetNumAvailableGPUs();
  LOG(INFO) << "Number of GPUs: " << num_gpus_;
  item_ = &item;
  graph_ = item.graph;
  LOG(INFO) << "Original graph size: " << graph_.node_size();
  if (item.fetch.empty()) {
    return Status(error::INVALID_ARGUMENT, "No fetch nodes provided.");
  }

  if (item.MainVariables().empty()) {
    return Status(error::INVALID_ARGUMENT, "No variables provided.");
  }

  for (const auto& init : item.init_ops) {
    VLOG(1) << "Init node: " << init;
  }

  for (const auto& fetch : item.fetch) {
    VLOG(1) << "Fetch node: " << fetch;
  }

  for (const auto& var : item.MainVariables()) {
    VLOG(2) << "Variable: " << var->name();
  }

  const std::set<string> apply_gradients_ops = {"ApplyGradientDescent",
                                                "ApplyProximalGradientDescent",
                                                "ApplyAdadelta",
                                                "ApplyAdagrad",
                                                "ApplyProximalAdagrad",
                                                "ApplyAdagradDA",
                                                "ApplyFtrl",
                                                "ApplyMomentum",
                                                "ApplyAdam",
                                                "ApplyRMSProp",
                                                "ApplyCenteredRMSProp"};
  for (int i = 0; i < graph_.node_size(); i++) {
    all_nodes_.insert(
        std::make_pair(graph_.node(i).name(), graph_.mutable_node(i)));
    if (apply_gradients_ops.find(graph_.node(i).op()) !=
        apply_gradients_ops.end()) {
      apply_gradients_nodes_.insert(graph_.node(i).name());
      VLOG(2) << "Apply gradients node: " << graph_.node(i).name();
    }
  }

  auto div_const_node = AddNodeDivConst();
  all_nodes_.insert(std::make_pair(div_const_node->name(), div_const_node));
  std::map<string, int> gradient_pos = {{"ApplyGradientDescent", 2},
                                        {"ApplyProximalGradientDescent", 4},
                                        {"ApplyAdadelta", 6},
                                        {"ApplyAdagrad", 3},
                                        {"ApplyProximalAdagrad", 5},
                                        {"ApplyAdagradDA", 3},
                                        {"ApplyFtrl", 3},
                                        {"ApplyMomentum", 3},
                                        {"ApplyAdam", 9},
                                        {"ApplyRMSProp", 7},
                                        {"ApplyCenteredRMSProp", 8}};
  for (const auto& apply_gradient_node_name : apply_gradients_nodes_) {
    auto apply_gradients_op = all_nodes_[apply_gradient_node_name]->op();
    auto apply_gradients_node = all_nodes_[apply_gradient_node_name];

    auto div_node = AddNodeDiv(
        apply_gradient_node_name,
        apply_gradients_node->input(gradient_pos[apply_gradients_op]),
        div_const_node->name());
    all_nodes_.insert(std::make_pair(div_node->name(), div_node));
    *apply_gradients_node->mutable_input(gradient_pos[apply_gradients_op]) =
        div_node->name();
  }
  LOG(INFO) << "Graph size after adding div nodes: " << all_nodes_.size();

  std::vector<const NodeDef*> train_nodes;
  TF_RETURN_IF_ERROR(ComputeTransitiveFanin(graph_, item.fetch, &train_nodes));
  LOG(INFO) << "Number of training nodes: " << train_nodes.size();

  const NodeDef* dequeue_node = nullptr;
  for (const auto& train_node : train_nodes) {
    if (IsDequeueOp(*train_node)) {
      dequeue_node = train_node;
      break;
    }
  }

  std::vector<const NodeDef*> input_nodes;
  if (dequeue_node) {
    LOG(INFO) << "Dequeue node: " << dequeue_node->name();
    TF_RETURN_IF_ERROR(ComputeTransitiveFanin(graph_, {dequeue_node->name()},
                                              {}, &input_nodes));
  }
  LOG(INFO) << "Number of input nodes: " << input_nodes.size();

  std::set<string> dont_replicate_nodes;
  for (const auto& variable : item.MainVariables()) {
    dont_replicate_nodes.insert(variable->name());
  }

  for (const auto& init : item.init_ops) {
    dont_replicate_nodes.insert(NodeName(init));
  }

  // Don't replicate all input nodes, except the dequeue node.
  for (const auto& input_node : input_nodes) {
    if (input_node->name() != dequeue_node->name()) {
      dont_replicate_nodes.insert(input_node->name());
    }
  }

  for (const auto& node : train_nodes) {
    if (dont_replicate_nodes.find(node->name()) == dont_replicate_nodes.end()) {
      replica_nodes_.insert(node->name());
    }
  }
  LOG(INFO) << "Number of replica nodes: " << replica_nodes_.size();

  for (const auto& node : all_nodes_) {
    if (replica_nodes_.find(node.first) == replica_nodes_.end()) {
      shared_nodes_.insert(node.first);
    }
  }
  LOG(INFO) << "Number of shared nodes: " << shared_nodes_.size();
  return Status::OK();
}


NodeDef* AutoParallel::AddNodeDiv(const string& name, const string& input_a,
                                  const string& input_b) {
  NodeDef* node = graph_.add_node();
  node->set_name(strings::StrCat(kAutoParallelPrefix, "-Div-", name));
  node->set_op("RealDiv");
  node->add_input(input_a);
  node->add_input(input_b);
  AttrValue attr_type;
  attr_type.set_type(DT_FLOAT);
  node->mutable_attr()->insert({"T", attr_type});
  return node;
}


[mod_wstunnel] fix crash with bad hybivers (fixes #3165)

(thx Michał Dardas)

x-ref:
  "mod_wstunnel null pointer dereference"
  https://redmine.lighttpd.net/issues/3165

static void wstunnel_handler_ctx_free(void *gwhctx) {
    handler_ctx *hctx = (handler_ctx *)gwhctx;
    chunk_buffer_release(hctx->frame.payload);
}


static int send_rfc_6455(handler_ctx *hctx, mod_wstunnel_frame_type_t type, const char *payload, size_t siz) {
    char mem[10];
    size_t len;

    /* allowed null payload for ping, pong, close frame */
    if (payload == NULL && (   type == MOD_WEBSOCKET_FRAME_TYPE_TEXT
                            || type == MOD_WEBSOCKET_FRAME_TYPE_BIN   )) {
        return -1;
    }

    switch (type) {
    case MOD_WEBSOCKET_FRAME_TYPE_TEXT:
        mem[0] = (char)(0x80 | MOD_WEBSOCKET_OPCODE_TEXT);
        DEBUG_LOG_DEBUG("%s", "type = text");
        break;
    case MOD_WEBSOCKET_FRAME_TYPE_BIN:
        mem[0] = (char)(0x80 | MOD_WEBSOCKET_OPCODE_BIN);
        DEBUG_LOG_DEBUG("%s", "type = binary");
        break;
    case MOD_WEBSOCKET_FRAME_TYPE_PING:
        mem[0] = (char) (0x80 | MOD_WEBSOCKET_OPCODE_PING);
        DEBUG_LOG_DEBUG("%s", "type = ping");
        break;
    case MOD_WEBSOCKET_FRAME_TYPE_PONG:
        mem[0] = (char)(0x80 | MOD_WEBSOCKET_OPCODE_PONG);
        DEBUG_LOG_DEBUG("%s", "type = pong");
        break;
    case MOD_WEBSOCKET_FRAME_TYPE_CLOSE:
    default:
        mem[0] = (char)(0x80 | MOD_WEBSOCKET_OPCODE_CLOSE);
        DEBUG_LOG_DEBUG("%s", "type = close");
        break;
    }

    DEBUG_LOG_DEBUG("payload size=%zx", siz);
    if (siz < MOD_WEBSOCKET_FRAME_LEN16) {
        mem[1] = siz;
        len = 2;
    }
    else if (siz <= UINT16_MAX) {
        mem[1] = MOD_WEBSOCKET_FRAME_LEN16;
        mem[2] = (siz >> 8) & 0xff;
        mem[3] = siz & 0xff;
        len = 1+MOD_WEBSOCKET_FRAME_LEN16_CNT+1;
    }
    else {
        mem[1] = MOD_WEBSOCKET_FRAME_LEN63;
        mem[2] = 0;
        mem[3] = 0;
        mem[4] = 0;
        mem[5] = 0;
        mem[6] = (siz >> 24) & 0xff;
        mem[7] = (siz >> 16) & 0xff;
        mem[8] = (siz >> 8) & 0xff;
        mem[9] = siz & 0xff;
        len = 1+MOD_WEBSOCKET_FRAME_LEN63_CNT+1;
    }
    request_st * const r = hctx->gw.r;
    http_chunk_append_mem(r, mem, len);
    if (siz) http_chunk_append_mem(r, payload, siz);
    DEBUG_LOG_DEBUG("send data to client (fd=%d), frame size=%zx",
                    r->con->fd, len+siz);
    return 0;
}


static handler_t wstunnel_handler_setup (request_st * const r, plugin_data * const p) {
    handler_ctx *hctx = r->plugin_ctx[p->id];
    int hybivers;
    hctx->errh = r->conf.errh;/*(for mod_wstunnel-specific DEBUG_* macros)*/
    hctx->conf = p->conf; /*(copies struct)*/
    hybivers = wstunnel_check_request(r, hctx);
    if (hybivers < 0) {
        r->handler_module = NULL;
        return HANDLER_FINISHED;
    }
    hctx->hybivers = hybivers;
    if (0 == hybivers) {
        DEBUG_LOG_INFO("WebSocket Version = %s", "hybi-00");
    }
    else {
        DEBUG_LOG_INFO("WebSocket Version = %d", hybivers);
    }

    hctx->gw.opts.backend     = BACKEND_PROXY; /*(act proxy-like)*/
    hctx->gw.opts.pdata       = hctx;
    hctx->gw.opts.parse       = wstunnel_recv_parse;
    hctx->gw.stdin_append     = wstunnel_stdin_append;
    hctx->gw.create_env       = wstunnel_create_env;
    hctx->gw.handler_ctx_free = wstunnel_handler_ctx_free;
    hctx->gw.backend_error    = wstunnel_backend_error;
    hctx->gw.response         = chunk_buffer_acquire();

    hctx->frame.state         = MOD_WEBSOCKET_FRAME_STATE_INIT;
    hctx->frame.ctl.siz       = 0;
    hctx->frame.payload       = chunk_buffer_acquire();

    unsigned int binary = hctx->conf.frame_type; /*(0 = "text"; 1 = "binary")*/
    if (!binary) {
        const buffer *vb =
          http_header_request_get(r, HTTP_HEADER_OTHER, CONST_STR_LEN("Sec-WebSocket-Protocol"));
        if (NULL != vb) {
            for (const char *s = vb->ptr; *s; ++s) {
                while (*s==' '||*s=='\t'||*s=='\r'||*s=='\n') ++s;
                if (buffer_eq_icase_ssn(s, CONST_STR_LEN("binary"))) {
                    s += sizeof("binary")-1;
                    while (*s==' '||*s=='\t'||*s=='\r'||*s=='\n') ++s;
                    if (*s==','||*s=='\0') {
                        hctx->subproto = 1;
                        binary = 1;
                        break;
                    }
                }
                else if (buffer_eq_icase_ssn(s, CONST_STR_LEN("base64"))) {
                    s += sizeof("base64")-1;
                    while (*s==' '||*s=='\t'||*s=='\r'||*s=='\n') ++s;
                    if (*s==','||*s=='\0') {
                        hctx->subproto = -1;
                        break;
                    }
                }
                s = strchr(s, ',');
                if (NULL == s) break;
            }
        }
    }

    if (binary) {
        DEBUG_LOG_INFO("%s", "will recv binary data from backend");
        hctx->frame.type         = MOD_WEBSOCKET_FRAME_TYPE_BIN;
        hctx->frame.type_before  = MOD_WEBSOCKET_FRAME_TYPE_BIN;
        hctx->frame.type_backend = MOD_WEBSOCKET_FRAME_TYPE_BIN;
    }
    else {
        DEBUG_LOG_INFO("%s", "will recv text data from backend");
        hctx->frame.type         = MOD_WEBSOCKET_FRAME_TYPE_TEXT;
        hctx->frame.type_before  = MOD_WEBSOCKET_FRAME_TYPE_TEXT;
        hctx->frame.type_backend = MOD_WEBSOCKET_FRAME_TYPE_TEXT;
    }

    return HANDLER_GO_ON;
}


static int get_key_number(uint32_t *ret, const buffer *b) {
    const char * const s = b->ptr;
    size_t j = 0;
    unsigned long n;
    uint32_t sp = 0;
    char tmp[10 + 1]; /* #define UINT32_MAX_STRLEN 10 */

    for (size_t i = 0, used = buffer_clen(b); i < used; ++i) {
        if (light_isdigit(s[i])) {
            tmp[j] = s[i];
            if (++j >= sizeof(tmp)) return -1;
        }
        else if (s[i] == ' ') ++sp; /* count num spaces */
    }
    tmp[j] = '\0';
    n = strtoul(tmp, NULL, 10);
    if (n > UINT32_MAX || 0 == sp || !light_isdigit(*tmp)) return -1;
    *ret = (uint32_t)n / sp;
    return 0;
}


TRIGGER_FUNC(mod_wstunnel_handle_trigger) {
    const plugin_data * const p = p_d;
    const unix_time64_t cur_ts = log_monotonic_secs + 1;

    gw_handle_trigger(srv, p_d);

    for (connection *con = srv->conns; con; con = con->next) {
        request_st * const r = &con->request;
        handler_ctx *hctx = r->plugin_ctx[p->id];
        if (NULL == hctx || r->handler_module != p->self)
            continue;

        if (hctx->gw.state != GW_STATE_WRITE && hctx->gw.state != GW_STATE_READ)
            continue;

        if (cur_ts - con->read_idle_ts > r->conf.max_read_idle) {
            DEBUG_LOG_INFO("timeout client (fd=%d)", con->fd);
            mod_wstunnel_frame_send(hctx,MOD_WEBSOCKET_FRAME_TYPE_CLOSE,NULL,0);
            gw_handle_request_reset(r, p_d);
            joblist_append(con);
            /* avoid server.c closing connection with error due to max_read_idle
             * (might instead run joblist after plugins_call_handle_trigger())*/
            con->read_idle_ts = cur_ts;
            continue;
        }

        if (0 != hctx->hybivers
            && hctx->conf.ping_interval > 0
            && (int32_t)hctx->conf.ping_interval + hctx->ping_ts < cur_ts) {
            hctx->ping_ts = cur_ts;
            mod_wstunnel_frame_send(hctx, MOD_WEBSOCKET_FRAME_TYPE_PING, CONST_STR_LEN("ping"));
            joblist_append(con);
            continue;
        }
    }

    return HANDLER_GO_ON;
}


static int get_key3(request_st * const r, char *buf, uint32_t bytes) {
    /* 8 bytes should have been sent with request
     * for draft-ietf-hybi-thewebsocketprotocol-00 */
    chunkqueue *cq = &r->reqbody_queue;
    /*(caller should ensure bytes available prior to calling this routine)*/
    /*assert(chunkqueue_length(cq) >= 8);*/
    /*assert(8 == bytes);*/
    return chunkqueue_read_data(cq, buf, bytes, r->conf.errh);
}


static void mod_wstunnel_merge_config(plugin_config * const pconf, const config_plugin_value_t *cpv) {
    do {
        mod_wstunnel_merge_config_cpv(pconf, cpv);
    } while ((++cpv)->k_id != -1);
}


int mod_wstunnel_plugin_init(plugin *p) {
    p->version           = LIGHTTPD_VERSION_ID;
    p->name              = "wstunnel";
    p->init              = mod_wstunnel_init;
    p->cleanup           = gw_free;
    p->set_defaults      = mod_wstunnel_set_defaults;
    p->handle_request_reset = gw_handle_request_reset;
    p->handle_uri_clean  = mod_wstunnel_check_extension;
    p->handle_subrequest = gw_handle_subrequest;
    p->handle_trigger    = mod_wstunnel_handle_trigger;
    p->handle_waitpid    = gw_handle_waitpid_cb;
    return 0;
}


static int recv_rfc_6455(handler_ctx *hctx) {
    request_st * const r = hctx->gw.r;
    chunkqueue *cq = &r->reqbody_queue;
    buffer *payload = hctx->frame.payload;
    DEBUG_LOG_DEBUG("recv data from client (fd=%d), size=%llx",
                    r->con->fd, (long long)chunkqueue_length(cq));
    for (chunk *c = cq->first; c; c = c->next) {
        char *frame = c->mem->ptr+c->offset;
        /*(chunk_remaining_length() on MEM_CHUNK)*/
        size_t flen = (size_t)(buffer_clen(c->mem) - c->offset);
        /*(FILE_CHUNK not handled, but might need to add support)*/
        force_assert(c->type == MEM_CHUNK);
        for (size_t i = 0; i < flen; ) {
            switch (hctx->frame.state) {
            case MOD_WEBSOCKET_FRAME_STATE_INIT:
                switch (frame[i] & 0x0f) {
                case MOD_WEBSOCKET_OPCODE_CONT:
                    DEBUG_LOG_DEBUG("%s", "type = continue");
                    hctx->frame.type = hctx->frame.type_before;
                    break;
                case MOD_WEBSOCKET_OPCODE_TEXT:
                    DEBUG_LOG_DEBUG("%s", "type = text");
                    hctx->frame.type = MOD_WEBSOCKET_FRAME_TYPE_TEXT;
                    hctx->frame.type_before = hctx->frame.type;
                    break;
                case MOD_WEBSOCKET_OPCODE_BIN:
                    DEBUG_LOG_DEBUG("%s", "type = binary");
                    hctx->frame.type = MOD_WEBSOCKET_FRAME_TYPE_BIN;
                    hctx->frame.type_before = hctx->frame.type;
                    break;
                case MOD_WEBSOCKET_OPCODE_PING:
                    DEBUG_LOG_DEBUG("%s", "type = ping");
                    hctx->frame.type = MOD_WEBSOCKET_FRAME_TYPE_PING;
                    break;
                case MOD_WEBSOCKET_OPCODE_PONG:
                    DEBUG_LOG_DEBUG("%s", "type = pong");
                    hctx->frame.type = MOD_WEBSOCKET_FRAME_TYPE_PONG;
                    break;
                case MOD_WEBSOCKET_OPCODE_CLOSE:
                    DEBUG_LOG_DEBUG("%s", "type = close");
                    hctx->frame.type = MOD_WEBSOCKET_FRAME_TYPE_CLOSE;
                    return -1;
                    break;
                default:
                    DEBUG_LOG_ERR("%s", "type is invalid");
                    return -1;
                    break;
                }
                i++;
                hctx->frame.state = MOD_WEBSOCKET_FRAME_STATE_READ_LENGTH;
                break;
            case MOD_WEBSOCKET_FRAME_STATE_READ_LENGTH:
                if ((frame[i] & 0x80) != 0x80) {
                    DEBUG_LOG_ERR("%s", "payload was not masked");
                    return -1;
                }
                hctx->frame.ctl.mask_cnt = 0;
                hctx->frame.ctl.siz = (uint64_t)(frame[i] & 0x7f);
                if (hctx->frame.ctl.siz == 0) {
                    DEBUG_LOG_DEBUG("specified payload size=%llx",
                                    (unsigned long long)hctx->frame.ctl.siz);
                    hctx->frame.state = MOD_WEBSOCKET_FRAME_STATE_READ_MASK;
                }
                else if (hctx->frame.ctl.siz == MOD_WEBSOCKET_FRAME_LEN16) {
                    hctx->frame.ctl.siz = 0;
                    hctx->frame.ctl.siz_cnt = MOD_WEBSOCKET_FRAME_LEN16_CNT;
                    hctx->frame.state =
                        MOD_WEBSOCKET_FRAME_STATE_READ_EX_LENGTH;
                }
                else if (hctx->frame.ctl.siz == MOD_WEBSOCKET_FRAME_LEN63) {
                    hctx->frame.ctl.siz = 0;
                    hctx->frame.ctl.siz_cnt = MOD_WEBSOCKET_FRAME_LEN63_CNT;
                    hctx->frame.state =
                        MOD_WEBSOCKET_FRAME_STATE_READ_EX_LENGTH;
                }
                else {
                    DEBUG_LOG_DEBUG("specified payload size=%llx",
                                    (unsigned long long)hctx->frame.ctl.siz);
                    hctx->frame.state = MOD_WEBSOCKET_FRAME_STATE_READ_MASK;
                }
                i++;
                break;
            case MOD_WEBSOCKET_FRAME_STATE_READ_EX_LENGTH:
                hctx->frame.ctl.siz =
                    (hctx->frame.ctl.siz << 8) + (frame[i] & 0xff);
                hctx->frame.ctl.siz_cnt--;
                if (hctx->frame.ctl.siz_cnt <= 0) {
                    if (hctx->frame.type == MOD_WEBSOCKET_FRAME_TYPE_PING &&
                        hctx->frame.ctl.siz > MOD_WEBSOCKET_BUFMAX) {
                        DEBUG_LOG_WARN("frame size has been exceeded: %x",
                                       MOD_WEBSOCKET_BUFMAX);
                        return -1;
                    }
                    DEBUG_LOG_DEBUG("specified payload size=%llx",
                                    (unsigned long long)hctx->frame.ctl.siz);
                    hctx->frame.state = MOD_WEBSOCKET_FRAME_STATE_READ_MASK;
                }
                i++;
                break;
            case MOD_WEBSOCKET_FRAME_STATE_READ_MASK:
                hctx->frame.ctl.mask[hctx->frame.ctl.mask_cnt] = frame[i];
                hctx->frame.ctl.mask_cnt++;
                if (hctx->frame.ctl.mask_cnt >= MOD_WEBSOCKET_MASK_CNT) {
                    hctx->frame.ctl.mask_cnt = 0;
                    if (hctx->frame.type == MOD_WEBSOCKET_FRAME_TYPE_PING &&
                        hctx->frame.ctl.siz == 0) {
                        mod_wstunnel_frame_send(hctx,
                                                MOD_WEBSOCKET_FRAME_TYPE_PONG,
                                                NULL, 0);
                    }
                    if (hctx->frame.ctl.siz == 0) {
                        hctx->frame.state = MOD_WEBSOCKET_FRAME_STATE_INIT;
                    }
                    else {
                        hctx->frame.state =
                            MOD_WEBSOCKET_FRAME_STATE_READ_PAYLOAD;
                    }
                }
                i++;
                break;
            case MOD_WEBSOCKET_FRAME_STATE_READ_PAYLOAD:
                /* hctx->frame.ctl.siz <= SIZE_MAX */
                if (hctx->frame.ctl.siz <= flen - i) {
                    DEBUG_LOG_DEBUG("read payload, size=%llx",
                                    (unsigned long long)hctx->frame.ctl.siz);
                    buffer_append_string_len(payload, frame+i, (size_t)
                                             (hctx->frame.ctl.siz & SIZE_MAX));
                    i += (size_t)(hctx->frame.ctl.siz & SIZE_MAX);
                    hctx->frame.ctl.siz = 0;
                    hctx->frame.state = MOD_WEBSOCKET_FRAME_STATE_INIT;
                    DEBUG_LOG_DEBUG("rest of frame size=%zx", flen - i);
                /* SIZE_MAX < hctx->frame.ctl.siz */
                }
                else {
                    DEBUG_LOG_DEBUG("read payload, size=%zx", flen - i);
                    buffer_append_string_len(payload, frame+i, flen - i);
                    hctx->frame.ctl.siz -= flen - i;
                    i += flen - i;
                    DEBUG_LOG_DEBUG("rest of payload size=%llx",
                                    (unsigned long long)hctx->frame.ctl.siz);
                }
                switch (hctx->frame.type) {
                case MOD_WEBSOCKET_FRAME_TYPE_TEXT:
                case MOD_WEBSOCKET_FRAME_TYPE_BIN:
                  {
                    unmask_payload(hctx);
                    chunkqueue_append_buffer(&hctx->gw.wb, payload);
                    /*buffer_clear(payload);*//*chunkqueue_append_buffer clear*/
                    break;
                  }
                case MOD_WEBSOCKET_FRAME_TYPE_PING:
                    if (hctx->frame.ctl.siz == 0) {
                        unmask_payload(hctx);
                        mod_wstunnel_frame_send(hctx,
                          MOD_WEBSOCKET_FRAME_TYPE_PONG,
                          payload->ptr, buffer_clen(payload));
                        buffer_clear(payload);
                    }
                    break;
                case MOD_WEBSOCKET_FRAME_TYPE_PONG:
                    buffer_clear(payload);
                    break;
                case MOD_WEBSOCKET_FRAME_TYPE_CLOSE:
                default:
                    DEBUG_LOG_ERR("%s", "BUG: invalid frame type");
                    return -1;
                }
                break;
            default:
                DEBUG_LOG_ERR("%s", "BUG: invalid state");
                return -1;
            }
        }
    }
    /* XXX: should add ability to handle and preserve partial frames above */
    /*(not chunkqueue_reset(); do not reset cq->bytes_in, cq->bytes_out)*/
    chunkqueue_mark_written(cq, chunkqueue_length(cq));
    return 0;
}


static void mod_wstunnel_patch_config(request_st * const r, plugin_data * const p) {
    memcpy(&p->conf, &p->defaults, sizeof(plugin_config));
    for (int i = 1, used = p->nconfig; i < used; ++i) {
        if (config_check_cond(r, (uint32_t)p->cvlist[i].k_id))
            mod_wstunnel_merge_config(&p->conf, p->cvlist+p->cvlist[i].v.u2[0]);
    }
}


static void mod_wstunnel_merge_config_cpv(plugin_config * const pconf, const config_plugin_value_t * const cpv) {
    switch (cpv->k_id) { /* index into static config_plugin_keys_t cpk[] */
      case 0: /* wstunnel.server */
        if (cpv->vtype == T_CONFIG_LOCAL) {
            gw_plugin_config * const gw = cpv->v.v;
            pconf->gw.exts      = gw->exts;
            pconf->gw.exts_auth = gw->exts_auth;
            pconf->gw.exts_resp = gw->exts_resp;
        }
        break;
      case 1: /* wstunnel.balance */
        /*if (cpv->vtype == T_CONFIG_LOCAL)*//*always true here for this param*/
            pconf->gw.balance = (int)cpv->v.u;
        break;
      case 2: /* wstunnel.debug */
        pconf->gw.debug = (int)cpv->v.u;
        break;
      case 3: /* wstunnel.map-extensions */
        pconf->gw.ext_mapping = cpv->v.a;
        break;
      case 4: /* wstunnel.frame-type */
        pconf->frame_type = cpv->v.u;
        break;
      case 5: /* wstunnel.origins */
        pconf->origins = cpv->v.a;
        break;
      case 6: /* wstunnel.ping-interval */
        pconf->ping_interval = cpv->v.shrt;
        break;
      default:/* should not happen */
        return;
    }
}


static int create_response_ietf_00(handler_ctx *hctx) {
    request_st * const r = hctx->gw.r;

    /* "Origin" header is preferred
     * ("Sec-WebSocket-Origin" is from older drafts of websocket spec) */
    const buffer *origin = http_header_request_get(r, HTTP_HEADER_OTHER, CONST_STR_LEN("Origin"));
    if (NULL == origin) {
        origin =
          http_header_request_get(r, HTTP_HEADER_OTHER, CONST_STR_LEN("Sec-WebSocket-Origin"));
    }
    if (NULL == origin) {
        DEBUG_LOG_ERR("%s", "Origin header is invalid");
        return -1;
    }
    if (!r->http_host || buffer_is_blank(r->http_host)) {
        DEBUG_LOG_ERR("%s", "Host header does not exist");
        return -1;
    }

    /* calc MD5 sum from keys */
    if (create_MD5_sum(r) < 0) {
        DEBUG_LOG_ERR("%s", "Sec-WebSocket-Key is invalid");
        return -1;
    }

    http_header_response_set(r, HTTP_HEADER_UPGRADE,
                             CONST_STR_LEN("Upgrade"),
                             CONST_STR_LEN("websocket"));
  #if 0 /*(added later in http_response_write_header())*/
    http_header_response_append(r, HTTP_HEADER_CONNECTION,
                                CONST_STR_LEN("Connection"),
                                CONST_STR_LEN("upgrade"));
  #endif
  #if 0 /*(Sec-WebSocket-Origin header is not required for hybi-00)*/
    /* Note: it is insecure to simply reflect back origin provided by client
     * (if admin did not configure restricted list of valid origins)
     * (see wstunnel_check_request()) */
    http_header_response_set(r, HTTP_HEADER_OTHER,
                             CONST_STR_LEN("Sec-WebSocket-Origin"),
                             BUF_PTR_LEN(origin));
  #endif

    buffer * const value =
      http_header_response_set_ptr(r, HTTP_HEADER_OTHER,
                                   CONST_STR_LEN("Sec-WebSocket-Location"));
    if (buffer_is_equal_string(&r->uri.scheme, CONST_STR_LEN("https")))
        buffer_copy_string_len(value, CONST_STR_LEN("wss://"));
    else
        buffer_copy_string_len(value, CONST_STR_LEN("ws://"));
    buffer_append_str2(value, BUF_PTR_LEN(r->http_host),
                              BUF_PTR_LEN(&r->uri.path));
    return 0;
}


static void wstunnel_backend_error(gw_handler_ctx *gwhctx) {
    handler_ctx *hctx = (handler_ctx *)gwhctx;
    if (hctx->gw.state == GW_STATE_WRITE || hctx->gw.state == GW_STATE_READ) {
        mod_wstunnel_frame_send(hctx, MOD_WEBSOCKET_FRAME_TYPE_CLOSE, CONST_STR_LEN("1001")); /* 1001 Going Away */
    }
}


static int wstunnel_check_request(request_st * const r, handler_ctx * const hctx) {
    const buffer * const vers =
      http_header_request_get(r, HTTP_HEADER_OTHER, CONST_STR_LEN("Sec-WebSocket-Version"));
    const long hybivers = (NULL != vers)
      ? light_isdigit(*vers->ptr) ? strtol(vers->ptr, NULL, 10) : -1
      : 0;
    if (hybivers < 0 || hybivers > INT_MAX) {
        DEBUG_LOG_ERR("%s", "invalid Sec-WebSocket-Version");
        r->http_status = 400; /* Bad Request */
        return -1;
    }

    /*(redundant since HTTP/1.1 required in mod_wstunnel_check_extension())*/
    if (!r->http_host || buffer_is_blank(r->http_host)) {
        DEBUG_LOG_ERR("%s", "Host header does not exist");
        r->http_status = 400; /* Bad Request */
        return -1;
    }

    if (!wstunnel_is_allowed_origin(r, hctx)) {
        return -1;
    }

    return (int)hybivers;
}


SETDEFAULTS_FUNC(mod_wstunnel_set_defaults) {
    static const config_plugin_keys_t cpk[] = {
      { CONST_STR_LEN("wstunnel.server"),
        T_CONFIG_ARRAY_KVARRAY,
        T_CONFIG_SCOPE_CONNECTION }
     ,{ CONST_STR_LEN("wstunnel.balance"),
        T_CONFIG_STRING,
        T_CONFIG_SCOPE_CONNECTION }
     ,{ CONST_STR_LEN("wstunnel.debug"),
        T_CONFIG_INT,
        T_CONFIG_SCOPE_CONNECTION }
     ,{ CONST_STR_LEN("wstunnel.map-extensions"),
        T_CONFIG_ARRAY_KVSTRING,
        T_CONFIG_SCOPE_CONNECTION }
     ,{ CONST_STR_LEN("wstunnel.frame-type"),
        T_CONFIG_STRING,
        T_CONFIG_SCOPE_CONNECTION }
     ,{ CONST_STR_LEN("wstunnel.origins"),
        T_CONFIG_ARRAY_VLIST,
        T_CONFIG_SCOPE_CONNECTION }
     ,{ CONST_STR_LEN("wstunnel.ping-interval"),
        T_CONFIG_SHORT,
        T_CONFIG_SCOPE_CONNECTION }
     ,{ NULL, 0,
        T_CONFIG_UNSET,
        T_CONFIG_SCOPE_UNSET }
    };

    plugin_data * const p = p_d;
    if (!config_plugin_values_init(srv, p, cpk, "mod_wstunnel"))
        return HANDLER_ERROR;

    /* process and validate config directives
     * (init i to 0 if global context; to 1 to skip empty global context) */
    for (int i = !p->cvlist[0].v.u2[1]; i < p->nconfig; ++i) {
        config_plugin_value_t *cpv = p->cvlist + p->cvlist[i].v.u2[0];
        gw_plugin_config *gw = NULL;
        for (; -1 != cpv->k_id; ++cpv) {
            switch (cpv->k_id) {
              case 0: /* wstunnel.server */
                gw = calloc(1, sizeof(gw_plugin_config));
                force_assert(gw);
                if (!gw_set_defaults_backend(srv, (gw_plugin_data *)p, cpv->v.a,
                                             gw, 0, cpk[cpv->k_id].k)) {
                    gw_plugin_config_free(gw);
                    return HANDLER_ERROR;
                }
                /* error if "mode" = "authorizer";
                 * wstunnel can not act as authorizer */
                /*(check after gw_set_defaults_backend())*/
                if (gw->exts_auth && gw->exts_auth->used) {
                    log_error(srv->errh, __FILE__, __LINE__,
                      "%s must not define any hosts with "
                      "attribute \"mode\" = \"authorizer\"", cpk[cpv->k_id].k);
                    gw_plugin_config_free(gw);
                    return HANDLER_ERROR;
                }
                cpv->v.v = gw;
                cpv->vtype = T_CONFIG_LOCAL;
                break;
              case 1: /* wstunnel.balance */
                cpv->v.u = (unsigned int)gw_get_defaults_balance(srv, cpv->v.b);
                break;
              case 2: /* wstunnel.debug */
              case 3: /* wstunnel.map-extensions */
                break;
              case 4: /* wstunnel.frame-type */
                /*(default frame-type to "text" unless "binary" is specified)*/
                cpv->v.u =
                  buffer_eq_icase_slen(cpv->v.b, CONST_STR_LEN("binary"));
                break;
              case 5: /* wstunnel.origins */
                for (uint32_t j = 0; j < cpv->v.a->used; ++j) {
                    buffer *origin = &((data_string *)cpv->v.a->data[j])->value;
                    if (buffer_is_blank(origin)) {
                        log_error(srv->errh, __FILE__, __LINE__,
                          "unexpected empty string in %s", cpk[cpv->k_id].k);
                        return HANDLER_ERROR;
                    }
                }
                break;
              case 6: /* wstunnel.ping-interval */
                break;
              default:/* should not happen */
                break;
            }
        }

        /* disable check-local for all exts (default enabled) */
        if (gw && gw->exts) { /*(check after gw_set_defaults_backend())*/
            gw_exts_clear_check_local(gw->exts);
        }
    }

    /* default is 0 */
    /*p->defaults.balance = (unsigned int)gw_get_defaults_balance(srv, NULL);*/
    p->defaults.ping_interval = 0; /* do not send ping */

    /* initialize p->defaults from global config context */
    if (p->nconfig > 0 && p->cvlist->v.u2[1]) {
        const config_plugin_value_t *cpv = p->cvlist + p->cvlist->v.u2[0];
        if (-1 != cpv->k_id)
            mod_wstunnel_merge_config(&p->defaults, cpv);
    }

    return HANDLER_GO_ON;
}


static handler_t wstunnel_recv_parse(request_st * const r, http_response_opts * const opts, buffer * const b, size_t n) {
    handler_ctx *hctx = (handler_ctx *)opts->pdata;
    DEBUG_LOG_DEBUG("recv data from backend (fd=%d), size=%zx", hctx->gw.fd, n);
    if (0 == n) return HANDLER_FINISHED;
    if (mod_wstunnel_frame_send(hctx,hctx->frame.type_backend,b->ptr,n) < 0) {
        DEBUG_LOG_ERR("%s", "fail to send data to client");
        return HANDLER_ERROR;
    }
    buffer_clear(b);
    UNUSED(r);
    return HANDLER_GO_ON;
}


static void unmask_payload(handler_ctx *hctx) {
    buffer * const b = hctx->frame.payload;
    for (size_t i = 0, used = buffer_clen(b); i < used; ++i) {
        b->ptr[i] ^= hctx->frame.ctl.mask[hctx->frame.ctl.mask_cnt];
        hctx->frame.ctl.mask_cnt = (hctx->frame.ctl.mask_cnt + 1) % 4;
    }
}


int mod_wstunnel_frame_send(handler_ctx *hctx, mod_wstunnel_frame_type_t type,
                             const char *payload, size_t siz) {
  #ifdef _MOD_WEBSOCKET_SPEC_RFC_6455_
    if (hctx->hybivers >= 8) return send_rfc_6455(hctx, type, payload, siz);
  #endif /* _MOD_WEBSOCKET_SPEC_RFC_6455_ */
  #ifdef _MOD_WEBSOCKET_SPEC_IETF_00_
    if (0 == hctx->hybivers) return send_ietf_00(hctx, type, payload, siz);
  #endif /* _MOD_WEBSOCKET_SPEC_IETF_00_ */
    return -1;
}


static int recv_ietf_00(handler_ctx *hctx) {
    request_st * const r = hctx->gw.r;
    chunkqueue *cq = &r->reqbody_queue;
    buffer *payload = hctx->frame.payload;
    char *mem;
    DEBUG_LOG_DEBUG("recv data from client (fd=%d), size=%llx",
                    r->con->fd, (long long)chunkqueue_length(cq));
    for (chunk *c = cq->first; c; c = c->next) {
        char *frame = c->mem->ptr+c->offset;
        /*(chunk_remaining_length() on MEM_CHUNK)*/
        size_t flen = (size_t)(buffer_clen(c->mem) - c->offset);
        /*(FILE_CHUNK not handled, but might need to add support)*/
        force_assert(c->type == MEM_CHUNK);
        for (size_t i = 0; i < flen; ) {
            switch (hctx->frame.state) {
            case MOD_WEBSOCKET_FRAME_STATE_INIT:
                hctx->frame.ctl.siz = 0;
                if (frame[i] == 0x00) {
                    hctx->frame.state = MOD_WEBSOCKET_FRAME_STATE_READ_PAYLOAD;
                    i++;
                }
                else if (((unsigned char *)frame)[i] == 0xff) {
                    DEBUG_LOG_DEBUG("%s", "recv close frame");
                    return -1;
                }
                else {
                    DEBUG_LOG_DEBUG("%s", "recv invalid frame");
                    return -1;
                }
                break;
            case MOD_WEBSOCKET_FRAME_STATE_READ_PAYLOAD:
                mem = (char *)memchr(frame+i, 0xff, flen - i);
                if (mem == NULL) {
                    DEBUG_LOG_DEBUG("got continuous payload, size=%zx", flen-i);
                    hctx->frame.ctl.siz += flen - i;
                    if (hctx->frame.ctl.siz > MOD_WEBSOCKET_BUFMAX) {
                        DEBUG_LOG_WARN("frame size has been exceeded: %x",
                                       MOD_WEBSOCKET_BUFMAX);
                        return -1;
                    }
                    buffer_append_string_len(payload, frame+i, flen - i);
                    i += flen - i;
                }
                else {
                    DEBUG_LOG_DEBUG("got final payload, size=%zx",
                                    mem - (frame+i));
                    hctx->frame.ctl.siz += (mem - (frame+i));
                    if (hctx->frame.ctl.siz > MOD_WEBSOCKET_BUFMAX) {
                        DEBUG_LOG_WARN("frame size has been exceeded: %x",
                                       MOD_WEBSOCKET_BUFMAX);
                        return -1;
                    }
                    buffer_append_string_len(payload, frame+i, mem - (frame+i));
                    i += (mem - (frame+i));
                    hctx->frame.state = MOD_WEBSOCKET_FRAME_STATE_INIT;
                }
                i++;
                if (hctx->frame.type == MOD_WEBSOCKET_FRAME_TYPE_TEXT
                    && !buffer_is_unset(payload)) { /*XXX: buffer_is_blank?*/
                    hctx->frame.ctl.siz = 0;
                    chunkqueue_append_buffer(&hctx->gw.wb, payload);
                    /*buffer_clear(payload);*//*chunkqueue_append_buffer clear*/
                }
                else {
                    if (hctx->frame.state == MOD_WEBSOCKET_FRAME_STATE_INIT
                        && !buffer_is_unset(payload)) {/*XXX: buffer_is_blank?*/
                        buffer *b;
                        size_t len = buffer_clen(payload);
                        len = (len+3)/4*3+1;
                        chunkqueue_get_memory(&hctx->gw.wb, &len);
                        b = hctx->gw.wb.last->mem;
                        len = buffer_clen(b);
                        DEBUG_LOG_DEBUG("try to base64 decode: %s",
                                        payload->ptr);
                        if (NULL ==
                            buffer_append_base64_decode(b, BUF_PTR_LEN(payload),
                                                        BASE64_STANDARD)) {
                            DEBUG_LOG_ERR("%s", "fail to base64-decode");
                            return -1;
                        }
                        buffer_clear(payload);
                        /*chunkqueue_use_memory()*/
                        hctx->gw.wb.bytes_in += buffer_clen(b)-len;
                    }
                }
                break;
            default: /* never reach */
                DEBUG_LOG_ERR("%s", "BUG: unknown state");
                return -1;
            }
        }
    }
    /* XXX: should add ability to handle and preserve partial frames above */
    /*(not chunkqueue_reset(); do not reset cq->bytes_in, cq->bytes_out)*/
    chunkqueue_mark_written(cq, chunkqueue_length(cq));
    return 0;
}


static handler_t wstunnel_create_env(gw_handler_ctx *gwhctx) {
    handler_ctx *hctx = (handler_ctx *)gwhctx;
    request_st * const r = hctx->gw.r;
    handler_t rc;
    if (0 == r->reqbody_length || r->http_version > HTTP_VERSION_1_1) {
        http_response_upgrade_read_body_unknown(r);
        chunkqueue_append_chunkqueue(&r->reqbody_queue, &r->read_queue);
    }
    rc = mod_wstunnel_handshake_create_response(hctx);
    if (rc != HANDLER_GO_ON) return rc;

    r->http_status = (r->http_version > HTTP_VERSION_1_1)
      ? 200  /* OK (response status for CONNECT) */
      : 101; /* Switching Protocols */
    r->resp_body_started = 1;

    hctx->ping_ts = log_monotonic_secs;
    gw_set_transparent(&hctx->gw);

    return HANDLER_GO_ON;
}


INIT_FUNC(mod_wstunnel_init) {
    return calloc(1, sizeof(plugin_data));
}


static handler_t wstunnel_stdin_append(gw_handler_ctx *gwhctx) {
    /* prepare websocket frames to backend */
    /* (caller should verify r->reqbody_queue) */
    /*assert(!chunkqueue_is_empty(&r->reqbody_queue));*/
    handler_ctx *hctx = (handler_ctx *)gwhctx;
    if (0 == mod_wstunnel_frame_recv(hctx))
        return HANDLER_GO_ON;
    else {
        /*(error)*/
        /* future: might differentiate client close request from client error,
         *         and then send 1000 or 1001 */
        request_st * const r = hctx->gw.r;
        DEBUG_LOG_INFO("disconnected from client (fd=%d)", r->con->fd);
        DEBUG_LOG_DEBUG("send close response to client (fd=%d)", r->con->fd);
        mod_wstunnel_frame_send(hctx, MOD_WEBSOCKET_FRAME_TYPE_CLOSE, CONST_STR_LEN("1000")); /* 1000 Normal Closure */
        gw_handle_request_reset(r, hctx->gw.plugin_data);
        return HANDLER_FINISHED;
    }
}


int mod_wstunnel_frame_recv(handler_ctx *hctx) {
  #ifdef _MOD_WEBSOCKET_SPEC_RFC_6455_
    if (hctx->hybivers >= 8) return recv_rfc_6455(hctx);
  #endif /* _MOD_WEBSOCKET_SPEC_RFC_6455_ */
  #ifdef _MOD_WEBSOCKET_SPEC_IETF_00_
    if (0 == hctx->hybivers) return recv_ietf_00(hctx);
  #endif /* _MOD_WEBSOCKET_SPEC_IETF_00_ */
    return -1;
}


static handler_t mod_wstunnel_check_extension(request_st * const r, void *p_d) {
    plugin_data *p = p_d;
    handler_t rc;

    if (NULL != r->handler_module)
        return HANDLER_GO_ON;
  if (r->http_version > HTTP_VERSION_1_1) {
    if (!r->h2_connect_ext)
        return HANDLER_GO_ON;
  }
  else {
    if (r->http_method != HTTP_METHOD_GET)
        return HANDLER_GO_ON;
    if (r->http_version != HTTP_VERSION_1_1)
        return HANDLER_GO_ON;

    /*
     * Connection: upgrade, keep-alive, ...
     * Upgrade: WebSocket, ...
     */
    const buffer *vb;
    vb = http_header_request_get(r, HTTP_HEADER_UPGRADE, CONST_STR_LEN("Upgrade"));
    if (NULL == vb
        || !http_header_str_contains_token(BUF_PTR_LEN(vb), CONST_STR_LEN("websocket")))
        return HANDLER_GO_ON;
    vb = http_header_request_get(r, HTTP_HEADER_CONNECTION, CONST_STR_LEN("Connection"));
    if (NULL == vb
        || !http_header_str_contains_token(BUF_PTR_LEN(vb), CONST_STR_LEN("upgrade")))
        return HANDLER_GO_ON;
  }

    mod_wstunnel_patch_config(r, p);
    if (NULL == p->conf.gw.exts) return HANDLER_GO_ON;

    rc = gw_check_extension(r, (gw_plugin_data *)p, 1, sizeof(handler_ctx));
    return (HANDLER_GO_ON == rc && r->handler_module == p->self)
      ? wstunnel_handler_setup(r, p)
      : rc;
}


static int send_ietf_00(handler_ctx *hctx, mod_wstunnel_frame_type_t type, const char *payload, size_t siz) {
    static const char head =  0; /* 0x00 */
    static const char tail = ~0; /* 0xff */
    request_st * const r = hctx->gw.r;
    char *mem;
    size_t len;

    switch (type) {
    case MOD_WEBSOCKET_FRAME_TYPE_TEXT:
        if (0 == siz) return 0;
        http_chunk_append_mem(r, &head, 1);
        http_chunk_append_mem(r, payload, siz);
        http_chunk_append_mem(r, &tail, 1);
        len = siz+2;
        break;
    case MOD_WEBSOCKET_FRAME_TYPE_BIN:
        if (0 == siz) return 0;
        http_chunk_append_mem(r, &head, 1);
        len = 4*(siz/3)+4+1;
        /* avoid accumulating too much data in memory; send to tmpfile */
        mem = malloc(len);
        force_assert(mem);
        len=li_to_base64(mem,len,(unsigned char *)payload,siz,BASE64_STANDARD);
        http_chunk_append_mem(r, mem, len);
        free(mem);
        http_chunk_append_mem(r, &tail, 1);
        len += 2;
        break;
    case MOD_WEBSOCKET_FRAME_TYPE_CLOSE:
        http_chunk_append_mem(r, &tail, 1);
        http_chunk_append_mem(r, &head, 1);
        len = 2;
        break;
    default:
        DEBUG_LOG_ERR("%s", "invalid frame type");
        return -1;
    }
    DEBUG_LOG_DEBUG("send data to client (fd=%d), frame size=%zx",
                    r->con->fd, len);
    return 0;
}


static int wstunnel_is_allowed_origin(request_st * const r, handler_ctx * const hctx) {
    /* If allowed origins is set (and not empty list), fail closed if no match.
     * Note that origin provided in request header has not been normalized, so
     * change in case or other non-normal forms might not match allowed list */
    const array * const allowed_origins = hctx->conf.origins;
    const buffer *origin = NULL;
    size_t olen;

    if (NULL == allowed_origins || 0 == allowed_origins->used) {
        DEBUG_LOG_INFO("%s", "allowed origins not specified");
        return 1;
    }

    /* "Origin" header is preferred
     * ("Sec-WebSocket-Origin" is from older drafts of websocket spec) */
    origin = http_header_request_get(r, HTTP_HEADER_OTHER, CONST_STR_LEN("Origin"));
    if (NULL == origin) {
        origin =
          http_header_request_get(r, HTTP_HEADER_OTHER, CONST_STR_LEN("Sec-WebSocket-Origin"));
    }
    olen = origin ? buffer_clen(origin) : 0;
    if (0 == olen) {
        DEBUG_LOG_ERR("%s", "Origin header is invalid");
        r->http_status = 400; /* Bad Request */
        return 0;
    }

    for (size_t i = 0; i < allowed_origins->used; ++i) {
        buffer *b = &((data_string *)allowed_origins->data[i])->value;
        size_t blen = buffer_clen(b);
        if ((olen > blen ? origin->ptr[olen-blen-1] == '.' : olen == blen)
            && 0 == memcmp(origin->ptr+olen-blen, b->ptr, blen)) {
            DEBUG_LOG_INFO("%s matches allowed origin: %s",origin->ptr,b->ptr);
            return 1;
        }
    }
    DEBUG_LOG_INFO("%s does not match any allowed origins", origin->ptr);
    r->http_status = 403; /* Forbidden */
    return 0;
}


static int create_MD5_sum(request_st * const r) {
    uint32_t buf[4]; /* MD5 binary hash len */

    const buffer *key1 =
      http_header_request_get(r, HTTP_HEADER_OTHER, CONST_STR_LEN("Sec-WebSocket-Key1"));
    const buffer *key2 =
      http_header_request_get(r, HTTP_HEADER_OTHER, CONST_STR_LEN("Sec-WebSocket-Key2"));

    if (NULL == key1 || get_key_number(buf+0, key1) < 0 ||
        NULL == key2 || get_key_number(buf+1, key2) < 0 ||
        get_key3(r, (char *)(buf+2), 2*sizeof(uint32_t)) < 0) {
        return -1;
    }
  #ifdef __BIG_ENDIAN__
  #define ws_htole32(s,u)\
    (s)[0]=((u)>>24);    \
    (s)[1]=((u)>>16);    \
    (s)[2]=((u)>>8);     \
    (s)[3]=((u))
    ws_htole32((unsigned char *)(buf+0), buf[0]);
    ws_htole32((unsigned char *)(buf+1), buf[1]);
  #endif
    /*(overwrite buf[] with result)*/
    MD5_once((unsigned char *)buf, buf, sizeof(buf));
    chunkqueue_append_mem(&r->write_queue, (char *)buf, sizeof(buf));
    return 0;
}


handler_t mod_wstunnel_handshake_create_response(handler_ctx *hctx) {
    request_st * const r = hctx->gw.r;
  #ifdef _MOD_WEBSOCKET_SPEC_RFC_6455_
    if (hctx->hybivers >= 8) {
        DEBUG_LOG_DEBUG("%s", "send handshake response");
        if (0 != create_response_rfc_6455(hctx)) {
            r->http_status = 400; /* Bad Request */
            return HANDLER_ERROR;
        }
        return HANDLER_GO_ON;
    }
  #endif /* _MOD_WEBSOCKET_SPEC_RFC_6455_ */

  #ifdef _MOD_WEBSOCKET_SPEC_IETF_00_
    if (hctx->hybivers == 0 && r->http_version == HTTP_VERSION_1_1) {
      #ifdef _MOD_WEBSOCKET_SPEC_IETF_00_
        /* 8 bytes should have been sent with request
         * for draft-ietf-hybi-thewebsocketprotocol-00 */
        chunkqueue *cq = &r->reqbody_queue;
        if (chunkqueue_length(cq) < 8)
            return HANDLER_WAIT_FOR_EVENT;
      #endif /* _MOD_WEBSOCKET_SPEC_IETF_00_ */

        DEBUG_LOG_DEBUG("%s", "send handshake response");
        if (0 != create_response_ietf_00(hctx)) {
            r->http_status = 400; /* Bad Request */
            return HANDLER_ERROR;
        }
        return HANDLER_GO_ON;
    }
  #endif /* _MOD_WEBSOCKET_SPEC_IETF_00_ */

    DEBUG_LOG_ERR("%s", "not supported WebSocket Version");
    r->http_status = 503; /* Service Unavailable */
    return HANDLER_ERROR;
}


static int create_response_rfc_6455(handler_ctx *hctx) {
    request_st * const r = hctx->gw.r;
  if (r->http_version == HTTP_VERSION_1_1) {
    SHA_CTX sha;
    unsigned char sha_digest[SHA_DIGEST_LENGTH];

    const buffer *value_wskey =
      http_header_request_get(r, HTTP_HEADER_OTHER, CONST_STR_LEN("Sec-WebSocket-Key"));
    if (NULL == value_wskey) {
        DEBUG_LOG_ERR("%s", "Sec-WebSocket-Key is invalid");
        return -1;
    }

    /* get SHA1 hash of key */
    /* refer: RFC-6455 Sec.1.3 Opening Handshake */
    SHA1_Init(&sha);
    SHA1_Update(&sha, (const unsigned char *)BUF_PTR_LEN(value_wskey));
    SHA1_Update(&sha, (const unsigned char *)CONST_STR_LEN("258EAFA5-E914-47DA-95CA-C5AB0DC85B11"));
    SHA1_Final(sha_digest, &sha);

    http_header_response_set(r, HTTP_HEADER_UPGRADE,
                             CONST_STR_LEN("Upgrade"),
                             CONST_STR_LEN("websocket"));
  #if 0 /*(added later in http_response_write_header())*/
    http_header_response_append(r, HTTP_HEADER_CONNECTION,
                                CONST_STR_LEN("Connection"),
                                CONST_STR_LEN("upgrade"));
  #endif

    buffer * const value =
      http_header_response_set_ptr(r, HTTP_HEADER_OTHER,
                                   CONST_STR_LEN("Sec-WebSocket-Accept"));
    buffer_append_base64_encode(value, sha_digest, SHA_DIGEST_LENGTH, BASE64_STANDARD);
  }

    if (hctx->frame.type == MOD_WEBSOCKET_FRAME_TYPE_BIN)
        http_header_response_set(r, HTTP_HEADER_OTHER,
                                 CONST_STR_LEN("Sec-WebSocket-Protocol"),
                                 CONST_STR_LEN("binary"));
    else if (-1 == hctx->subproto)
        http_header_response_set(r, HTTP_HEADER_OTHER,
                                 CONST_STR_LEN("Sec-WebSocket-Protocol"),
                                 CONST_STR_LEN("base64"));

    return 0;
}


Fix FPE issue with `tf.raw_ops.Reverse`.

PiperOrigin-RevId: 371176973
Change-Id: Ic6d483bfc95313ec2299c2d1c956cfe96c96626c

void HandleReverseCase(OpKernelContext* context,
                       typename TTypes<bool, 1>::ConstTensor dims,
                       Tensor* result) {
  const Tensor& input = context->input(0);

  // Use optimized reverse if possible.
  if (NDIMS == 3 && std::is_same<Device, CPUDevice>::value &&
      data_type_can_memcpy<T>::value && (!dims(0) && dims(1) && !dims(2))) {
    if (input.dim_size(2) == 3) {
      DoHandleReverseCase<T, 3>(context, input, result);
    } else {
      DoHandleReverseCase<T, -1>(context, input, result);
    }
    return;
  }
  typename Eigen::array<bool, NDIMS> axes_di;
  for (int i = 0; i < NDIMS; i++) {
    axes_di[i] = dims(i);
  }
  functor::Reverse<Device, T, NDIMS>()(context->eigen_device<Device>(),
                                       input.tensor<T, NDIMS>(), axes_di,
                                       result->tensor<T, NDIMS>());
}

TensorFlow is an end-to-end open source platform for machine learning. An attacker can cause a denial of service via a FPE runtime error in `tf.raw_ops.Reverse`. This is because the implementation(https://github.com/tensorflow/tensorflow/blob/36229ea9e9451dac14a8b1f4711c435a1d84a594/tensorflow/core/kernels/reverse_op.cc#L75-L76) performs a division based on the first dimension of the tensor argument. The fix will be included in TensorFlow 2.5.0. We will also cherrypick this commit on TensorFlow 2.4.2, TensorFlow 2.3.3, TensorFlow 2.2.3 and TensorFlow 2.1.4, as these are also affected and still in supported range.


  void Compute(OpKernelContext* context) override {
    const Tensor& input = context->input(0);
    const Tensor& sparse_dims = context->input(1);

    if (TensorShapeUtils::IsScalar(input.shape()) || input.NumElements() == 0) {
      context->set_output(0, input);
    } else {
      const int input_dims = input.dims();
      const TensorShape& sparse_dims_shape = sparse_dims.shape();
      const auto& axes_sparse_flat = sparse_dims.flat<Tidx>();

      OP_REQUIRES(context, TensorShapeUtils::IsVector(sparse_dims_shape),
                  errors::InvalidArgument("'dims' must be 1-dimension, not ",
                                          sparse_dims.dims()));
      gtl::InlinedVector<bool, 8> axes_dense(input_dims, false);
      for (int dummy = 0; dummy < axes_sparse_flat.size(); dummy++) {
        Tidx axis = internal::SubtleMustCopy<Tidx>(axes_sparse_flat(dummy));
        Tidx canonical_axis = axis < 0 ? input_dims + axis : axis;
        OP_REQUIRES(context, canonical_axis >= 0 && canonical_axis < input_dims,
                    errors::InvalidArgument("'axis'[", dummy, "] = ", axis,
                                            " is out of valid range [", 0, ", ",
                                            input_dims - 1));
        OP_REQUIRES(context, !axes_dense[canonical_axis],
                    errors::InvalidArgument("axis ", canonical_axis,
                                            " specified more than once."));
        axes_dense[canonical_axis] = true;
      }

      OP_REQUIRES(context, input_dims <= 8,
                  errors::Unimplemented(
                      "reverse is not implemented for tensors of rank > 8."));

      Tensor* output = nullptr;
      OP_REQUIRES_OK(context,
                     context->allocate_output(0, input.shape(), &output));

      // TODO(cwhipkey): we can do dimension folding to reduce, e.g., a reverse
      // of a single dimension to the dims=3 or dims=2 case, regardless of the
      // number of dimensions in the tensor. This would let some ops use faster
      // lower-dimension code (and use optimized versions).

#define HANDLE_REVERSE(NDIMS)                                           \
  case NDIMS:                                                           \
    HandleReverseV2Case<Device, T, NDIMS>(context, axes_dense, output); \
    return;

      switch (input_dims) {
        HANDLE_REVERSE(0);
        HANDLE_REVERSE(1);
        HANDLE_REVERSE(2);
        HANDLE_REVERSE(3);
        HANDLE_REVERSE(4);
        HANDLE_REVERSE(5);
        HANDLE_REVERSE(6);
        HANDLE_REVERSE(7);
        HANDLE_REVERSE(8);
      }
#undef HANDLE_REVERSE
    }
  }


explicit ReverseOp(OpKernelConstruction* context) : OpKernel(context) {}


DoHandleReverseCase(OpKernelContext* context, const Tensor& input,
                    Tensor* result) {}


  void Compute(OpKernelContext* context) override {
    const Tensor& input = context->input(0);
    // If input is provided, check to make sure the first dimension is valid.
    if (input.dims() > 0) {
      OP_REQUIRES(
          context, input.dim_size(0) != 0,
          errors::InvalidArgument("Invalid input first dimension. Found 0."));
    }
    const Tensor& dims = context->input(1);

    if (TensorShapeUtils::IsScalar(input.shape())) {
      context->set_output(0, input);
    } else {
      const int input_dims = input.dims();
      OP_REQUIRES(context, TensorShapeUtils::IsVector(dims.shape()),
                  errors::InvalidArgument("'dims' must be 1-dimension, not ",
                                          dims.dims()));

      OP_REQUIRES(
          context, input_dims == dims.dim_size(0),
          errors::InvalidArgument(
              "'dims' must have the same number of values as 'input' has "
              "dimensions. 'input' has ",
              input_dims, "'dims' has ", dims.dim_size(0), " values"));
      OP_REQUIRES(context, input_dims <= 8,
                  errors::Unimplemented(
                      "reverse is not implemented for tensors of rank > 8."));

      Tensor* output = nullptr;
      OP_REQUIRES_OK(context,
                     context->allocate_output(0, input.shape(), &output));

#define HANDLE_REVERSE(NDIMS)                                               \
  case NDIMS:                                                               \
    HandleReverseCase<Device, T, NDIMS>(context, dims.vec<bool>(), output); \
    return;

      switch (input_dims) {
        HANDLE_REVERSE(0);
        HANDLE_REVERSE(1);
        HANDLE_REVERSE(2);
        HANDLE_REVERSE(3);
        HANDLE_REVERSE(4);
        HANDLE_REVERSE(5);
        HANDLE_REVERSE(6);
        HANDLE_REVERSE(7);
        HANDLE_REVERSE(8);
      }
#undef HANDLE_REVERSE
    }
  }


void ReverseRows(OpKernelContext* context, const Tensor& input,
                 Tensor* result) {
  auto work = [&input, result](int64 start, int64 end) {
    const int64 inner_size =
        NUM_CHANNELS > 0 ? NUM_CHANNELS : input.dim_size(2);
    const int64 middle_size = input.dim_size(1);
    const int64 row_size = inner_size * middle_size;
    DCHECK_EQ(input.dim_size(2), inner_size);

    const T* in_ptr = input.bit_casted_tensor<T, 3>().data();
    T* out_ptr = result->bit_casted_tensor<T, 3>().data();

    in_ptr += start * row_size;
    out_ptr += start * row_size;

    for (int outer_dim = start; outer_dim < end; ++outer_dim) {
      out_ptr += row_size;
      int remaining = middle_size;
      while (remaining > 0) {
        out_ptr -= inner_size;
        memcpy(out_ptr, in_ptr, inner_size * sizeof(T));
        in_ptr += inner_size;
        --remaining;
      }

      out_ptr += row_size;
    }
  };

  // Shard across outer dimension.
  const int64 N = input.dim_size(0);
  const int64 cost_per_unit = input.NumElements() / N;
  auto worker_threads = context->device()->tensorflow_cpu_worker_threads();
  Shard(worker_threads->num_threads, worker_threads->workers, N, cost_per_unit,
        std::move(work));
}


explicit ReverseV2Op(OpKernelConstruction* context) : OpKernel(context) {}


DoHandleReverseCase(OpKernelContext* context, const Tensor& input,
                    Tensor* result) {
  if (sizeof(T) == 1) {
    static_assert(sizeof(uint8) == 1, "uint8 must be 1 byte.");
    ReverseRows<uint8, NUM_CHANNELS>(context, input, result);
  } else if (sizeof(T) == 2) {
    static_assert(sizeof(uint16) == 2, "uint16 must be 2 bytes");
    ReverseRows<uint16, NUM_CHANNELS>(context, input, result);
  } else if (sizeof(T) == 4) {
    static_assert(sizeof(uint32) == 4, "uint32 must be 4 bytes");
    ReverseRows<uint32, NUM_CHANNELS>(context, input, result);
  } else if (sizeof(T) == 8) {
    static_assert(sizeof(uint64) == 8, "uint64 must be 8 bytes");
    ReverseRows<uint64, NUM_CHANNELS>(context, input, result);
  } else if (sizeof(T) == 16) {
    static_assert(sizeof(complex128) == 16, "complex128 must be 16 bytes");
    ReverseRows<complex128, NUM_CHANNELS>(context, input, result);
  } else {
    context->CtxFailure(errors::InvalidArgument(DataTypeString(input.dtype()),
                                                " has unexpected size of ",
                                                sizeof(T), " bytes"));
  }
}


void HandleReverseV2Case(OpKernelContext* context,
                         const gtl::ArraySlice<bool> axes, Tensor* result) {
  const Tensor& input = context->input(0);

  // Use optimized reverse if possible.
  if (NDIMS == 3 && std::is_same<Device, CPUDevice>::value &&
      data_type_can_memcpy<T>::value && (!axes[0] && axes[1] && !axes[2])) {
    if (input.dim_size(2) == 3) {
      DoHandleReverseCase<T, 3>(context, input, result);
    } else {
      DoHandleReverseCase<T, -1>(context, input, result);
    }
    return;
  }

  typename Eigen::array<bool, NDIMS> axes_di;
  for (int i = 0; i < NDIMS; i++) {
    axes_di[i] = axes[i];
  }
  functor::Reverse<Device, T, NDIMS>()(context->eigen_device<Device>(),
                                       input.tensor<T, NDIMS>(), axes_di,
                                       result->tensor<T, NDIMS>());
}


Fix NPE in restoring code.

PiperOrigin-RevId: 388303253
Change-Id: Ia8c68568cb854bca538909a182b31a618d68ce55

void RestoreTensor(OpKernelContext* context,
                   checkpoint::TensorSliceReader::OpenTableFunction open_func,
                   int preferred_shard, bool restore_slice, int restore_index) {
  const Tensor& file_pattern_t = context->input(0);
  {
    const int64_t size = file_pattern_t.NumElements();
    OP_REQUIRES(
        context, size == 1,
        errors::InvalidArgument(
            "Input 0 (file_pattern) must be a string scalar; got a tensor of ",
            size, " elements"));
  }
  const string& file_pattern = file_pattern_t.flat<tstring>()(0);

  const Tensor& tensor_name_t = context->input(1);
  {
    const int64_t size = tensor_name_t.NumElements();
    OP_REQUIRES(context, size > restore_index,
                errors::InvalidArgument(
                    "Input 1 (file_pattern) must be a have at least ",
                    restore_index + 1, " elements"));
  }
  const string& tensor_name = tensor_name_t.flat<tstring>()(restore_index);

  // If we cannot find a cached reader we will allocate our own.
  std::unique_ptr<checkpoint::TensorSliceReader> allocated_reader;

  const checkpoint::TensorSliceReader* reader = nullptr;

  if (context->slice_reader_cache()) {
    reader = context->slice_reader_cache()->GetReader(file_pattern, open_func,
                                                      preferred_shard);
  }
  if (!reader) {
    allocated_reader.reset(new checkpoint::TensorSliceReader(
        file_pattern, open_func, preferred_shard));
    reader = allocated_reader.get();
  }
  OP_REQUIRES_OK(context, CHECK_NOTNULL(reader)->status());

  // Get the shape and type from the save file.
  DataType type;
  TensorShape saved_shape;
  OP_REQUIRES(
      context, reader->HasTensor(tensor_name, &saved_shape, &type),
      errors::NotFound("Tensor name \"", tensor_name,
                       "\" not found in checkpoint files ", file_pattern));
  OP_REQUIRES(
      context, type == context->expected_output_dtype(restore_index),
      errors::InvalidArgument("Expected to restore a tensor of type ",
                              DataTypeString(context->expected_output_dtype(0)),
                              ", got a tensor of type ", DataTypeString(type),
                              " instead: tensor_name = ", tensor_name));

  // Shape of the output and slice to load.
  TensorShape output_shape(saved_shape);
  TensorSlice slice_to_load(saved_shape.dims());
  if (restore_slice) {
    const tstring& shape_spec =
        context->input(2).flat<tstring>()(restore_index);
    if (!shape_spec.empty()) {
      TensorShape parsed_shape;
      OP_REQUIRES_OK(context, checkpoint::ParseShapeAndSlice(
                                  shape_spec, &parsed_shape, &slice_to_load,
                                  &output_shape));
      OP_REQUIRES(
          context, parsed_shape.IsSameSize(saved_shape),
          errors::InvalidArgument(
              "Shape in shape_and_slice spec does not match the shape in the "
              "save file: ",
              parsed_shape.DebugString(),
              ", save file shape: ", saved_shape.DebugString()));
    }
  }

  Tensor* t = nullptr;
  OP_REQUIRES_OK(context,
                 context->allocate_output(restore_index, output_shape, &t));

  if (output_shape.num_elements() == 0) return;

#define READER_COPY(T)                                                \
  case DataTypeToEnum<T>::value:                                      \
    OP_REQUIRES(context,                                              \
                reader->CopySliceData(tensor_name, slice_to_load,     \
                                      t->flat<T>().data()),           \
                errors::InvalidArgument("Error copying slice data")); \
    break;

  switch (type) {
    TF_CALL_SAVE_RESTORE_TYPES(READER_COPY)
    default:
      context->SetStatus(errors::Unimplemented(
          "Restoring data type ", DataTypeString(type), " not yet supported"));
  }
#undef READER_COPY
}

TensorFlow is an end-to-end open source platform for machine learning. When restoring tensors via raw APIs, if the tensor name is not provided, TensorFlow can be tricked into dereferencing a null pointer. Alternatively, attackers can read memory outside the bounds of heap allocated data by providing some tensor names but not enough for a successful restoration. The [implementation](https://github.com/tensorflow/tensorflow/blob/47a06f40411a69c99f381495f490536972152ac0/tensorflow/core/kernels/save_restore_tensor.cc#L158-L159) retrieves the tensor list corresponding to the `tensor_name` user controlled input and immediately retrieves the tensor at the restoration index (controlled via `preferred_shard` argument). This occurs without validating that the provided list has enough values. If the list is empty this results in dereferencing a null pointer (undefined behavior). If, however, the list has some elements, if the restoration index is outside the bounds this results in heap OOB read. We have patched the issue in GitHub commit 9e82dce6e6bd1f36a57e08fa85af213e2b2f2622. The fix will be included in TensorFlow 2.6.0. We will also cherrypick this commit on TensorFlow 2.5.1, TensorFlow 2.4.3, and TensorFlow 2.3.4, as these are also affected and still in supported range.


  void run_with_new_reader() {
    BundleReader reader(Env::Default(), reader_prefix);
    if (!reader.status().ok()) {
      status = reader.status();
      return;
    }

    status = run(&reader);
  }


  bool should_run_in_pool(BundleReader* reader) const {
    TensorShape restored_full_shape;

    // Ignore status here; we'll catch the error later.
    if (!reader->LookupTensorShape(tensor_name, &restored_full_shape).ok()) {
      return false;
    }

    return restored_full_shape.num_elements() > kLargeShapeThreshold;
  }


  Status run(BundleReader* reader) {
    TensorShape restored_full_shape;
    TF_RETURN_IF_ERROR(
        reader->LookupTensorShape(tensor_name, &restored_full_shape));

    VLOG(1) << "Restoring tensor " << idx << " : " << tensor_name << " : "
            << restored_full_shape.num_elements();
    Tensor* restored_tensor;
    if (shape_and_slice.empty()) {
      // Lookup the full tensor.
      TF_RETURN_IF_ERROR(
          context->allocate_output(idx, restored_full_shape, &restored_tensor));
      TF_RETURN_IF_ERROR(reader->Lookup(tensor_name, restored_tensor));
    } else {
      // Lookup the slice.
      TensorShape parsed_full_shape;
      TensorSlice parsed_slice;
      TensorShape parsed_slice_shape;

      TF_RETURN_IF_ERROR(
          checkpoint::ParseShapeAndSlice(shape_and_slice, &parsed_full_shape,
                                         &parsed_slice, &parsed_slice_shape));

      if (!restored_full_shape.IsSameSize(parsed_full_shape)) {
        return errors::InvalidArgument(
            "tensor_name = ", tensor_name, "; shape in shape_and_slice spec ",
            parsed_full_shape.DebugString(),
            " does not match the shape stored in checkpoint: ",
            restored_full_shape.DebugString());
      }
      TF_RETURN_IF_ERROR(
          context->allocate_output(idx, parsed_slice_shape, &restored_tensor));
      TF_RETURN_IF_ERROR(
          reader->LookupSlice(tensor_name, parsed_slice, restored_tensor));
    }
    if (VLOG_IS_ON(5)) {
      if (restored_tensor->dtype() == DT_FLOAT) {
        const float* t_data = restored_tensor->flat<float>().data();
        float min = std::numeric_limits<float>::infinity();
        float max = -std::numeric_limits<float>::infinity();
        double avg = 0.0;
        for (int i = 0; i < restored_tensor->NumElements(); ++i) {
          if (t_data[i] < min) min = t_data[i];
          if (t_data[i] > max) max = t_data[i];
          avg += t_data[i];
        }
        VLOG(5) << " min " << min << " max " << max << " avg "
                << avg / restored_tensor->NumElements() << " total elts "
                << restored_tensor->NumElements();
      }
    }
    VLOG(1) << "Done restoring tensor " << idx << " : " << tensor_name << " : "
            << restored_full_shape.num_elements();
    return Status::OK();
  }


void SaveTensors(
    OpKernelContext* context,
    checkpoint::TensorSliceWriter::CreateBuilderFunction builder_func,
    bool save_slices) {
  const Tensor& filename_t = context->input(0);
  {
    const int64_t size = filename_t.NumElements();
    OP_REQUIRES(
        context, size == 1,
        errors::InvalidArgument(
            "Input 0 (filename) must be a string scalar; got a tensor of ",
            size, "elements"));
  }

  // Path, names, and slices if save_slices is true.
  const int kFixedInputs = save_slices ? 3 : 2;
  const Tensor& tensor_names_t = context->input(1);
  OP_REQUIRES(context,
              FastBoundsCheck(tensor_names_t.NumElements() + kFixedInputs,
                              std::numeric_limits<int>::max()),
              errors::InvalidArgument("Too many inputs to SaveTensors"));
  const int N = static_cast<int>(tensor_names_t.NumElements());
  const tstring* tensor_shapes_and_slices_ptr = nullptr;
  if (save_slices) {
    const Tensor& tensor_shapes_and_slices_t = context->input(2);
    OP_REQUIRES(
        context,
        tensor_shapes_and_slices_t.NumElements() == static_cast<int64>(N),
        errors::InvalidArgument("Expected ", N,
                                " elements for the tensor "
                                "shapes and slices but got ",
                                tensor_shapes_and_slices_t.NumElements()));
    tensor_shapes_and_slices_ptr =
        tensor_shapes_and_slices_t.flat<tstring>().data();
  }
  OP_REQUIRES(context, context->num_inputs() == N + kFixedInputs,
              errors::InvalidArgument("Expected totally ", N + kFixedInputs,
                                      " inputs as input #1 (which is a string "
                                      "tensor of saved names) contains ",
                                      N, " names, but received ",
                                      context->num_inputs(), " inputs"));

  VLOG(1) << "About to save tensors to file " << filename_t.flat<tstring>()(0)
          << "...";
  checkpoint::TensorSliceWriter writer(filename_t.flat<tstring>()(0),
                                       std::move(builder_func));

  Status s;
  auto tensor_names_flat = tensor_names_t.flat<tstring>();

  // Process tensors in sorted name order.  This allows us to avoid seeking
  // during restoration in the common case where we are restoring a full
  // checkpoint.
  std::vector<size_t> sorted_name_idx(tensor_names_flat.size());
  std::iota(sorted_name_idx.begin(), sorted_name_idx.end(), 0);
  std::sort(sorted_name_idx.begin(), sorted_name_idx.end(),
            [&tensor_names_flat](size_t a, size_t b) {
              return tensor_names_flat(a) < tensor_names_flat(b);
            });

  for (const size_t i : sorted_name_idx) {
    const string& name = tensor_names_flat(i);
    const Tensor& input = context->input(i + kFixedInputs);
    TensorShape shape(input.shape());
    TensorSlice slice(input.dims());
    if (save_slices && !tensor_shapes_and_slices_ptr[i].empty()) {
      const tstring& shape_spec = tensor_shapes_and_slices_ptr[i];
      TensorShape slice_shape;
      OP_REQUIRES_OK(context, checkpoint::ParseShapeAndSlice(
                                  shape_spec, &shape, &slice, &slice_shape));
      OP_REQUIRES(context, slice_shape.IsSameSize(input.shape()),
                  errors::InvalidArgument(
                      "Slice in shape_and_slice "
                      "specification does not match the "
                      "shape of the tensor to  save: ",
                      shape_spec, ", tensor: ", input.shape().DebugString()));
    }

#define WRITER_ADD(T)                                           \
  case DataTypeToEnum<T>::value:                                \
    s = writer.Add(name, shape, slice, input.flat<T>().data()); \
    break;

    switch (input.dtype()) {
      TF_CALL_SAVE_RESTORE_TYPES(WRITER_ADD)
      default:
        context->SetStatus(errors::Unimplemented("Saving data type ",
                                                 DataTypeString(input.dtype()),
                                                 " not yet supported"));
        return;
    }
#undef WRITER_ADD
    if (!s.ok()) {
      context->SetStatus(s);
      return;
    }
  }

  s = writer.Finish();
  if (!s.ok()) {
    context->SetStatus(s);
  }
}


Status RestoreTensorsV2(OpKernelContext* context, const Tensor& prefix,
                        const Tensor& tensor_names,
                        const Tensor& shape_and_slices,
                        gtl::ArraySlice<DataType> dtypes) {
  const string& prefix_string = prefix.scalar<tstring>()();

  const auto& tensor_names_flat = tensor_names.flat<tstring>();
  const auto& shape_and_slices_flat = shape_and_slices.flat<tstring>();

  // Sort lookup keys to improve locality when reading multiple tensors.
  std::vector<size_t> sorted_name_idx(tensor_names_flat.size());
  std::iota(sorted_name_idx.begin(), sorted_name_idx.end(), 0);
  std::sort(sorted_name_idx.begin(), sorted_name_idx.end(),
            [&tensor_names_flat](size_t a, size_t b) {
              return tensor_names_flat(a) < tensor_names_flat(b);
            });

  std::vector<std::unique_ptr<RestoreOp> > pool_restore_ops;
  std::vector<std::unique_ptr<RestoreOp> > direct_restore_ops;

  BundleReader default_reader(Env::Default(), prefix_string);
  TF_RETURN_IF_ERROR(default_reader.status());

  std::vector<string> mismatched_errors;
  for (const size_t i : sorted_name_idx) {
    TensorShape restored_full_shape;
    DataType original_dtype;
    const string& tensor_name = tensor_names_flat(i);
    TF_RETURN_IF_ERROR(default_reader.LookupDtypeAndShape(
        tensor_name, &original_dtype, &restored_full_shape));
    if (dtypes[i] != original_dtype) {
      string error_msg = strings::StrCat(
          "tensor_name = ", tensor_name, "; expected dtype ",
          DataTypeString(dtypes[i]), " does not equal original dtype ",
          DataTypeString(original_dtype));
      mismatched_errors.emplace_back(error_msg);
    }
  }
  if (!mismatched_errors.empty()) {
    const string error_msg = absl::StrJoin(mismatched_errors, "\n");
    return errors::InvalidArgument(error_msg);
  }

  for (auto i : sorted_name_idx) {
    const string& tensor_name = tensor_names_flat(i);
    const string& shape_and_slice = shape_and_slices_flat(i);
    auto op =
        new RestoreOp{context, i, tensor_name, shape_and_slice, prefix_string};
    if (op->should_run_in_pool(&default_reader)) {
      pool_restore_ops.emplace_back(op);
    } else {
      direct_restore_ops.emplace_back(op);
    }
  }

  {
    // Schedule any threaded operations first, skipping thread pool creation if
    // we don't have any expensive operations.
    std::unique_ptr<thread::ThreadPool> reader_pool;
    if (!pool_restore_ops.empty()) {
      reader_pool.reset(
          new thread::ThreadPool(Env::Default(), "restore_tensors", 8));
      for (auto& op : pool_restore_ops) {
        reader_pool->Schedule([&op]() { op->run_with_new_reader(); });
      }
    }

    // Read small tensors from the op thread
    for (auto& op : direct_restore_ops) {
      TF_RETURN_IF_ERROR(op->run(&default_reader));
    }
  }

  // Check status of pool ops; this must come after the pool shuts down.
  for (auto& op : pool_restore_ops) {
    TF_RETURN_IF_ERROR(op->status);
  }

  for (auto i : sorted_name_idx) {
    const string& tensor_name = tensor_names_flat(i);
    if (dtypes[i] != context->mutable_output(i)->dtype()) {
      return errors::InvalidArgument(
          "tensor_name = ", tensor_name, "; expected dtype ",
          DataTypeString(dtypes[i]), " does not equal restored dtype ",
          DataTypeString(context->mutable_output(i)->dtype()));
    }
  }

  return Status::OK();
}


avformat/mov: Check count sums in build_open_gop_key_points()

Fixes: ffmpeg.md
Fixes: Out of array access
Fixes: CVE-2022-2566

Found-by: Andy Nguyen <theflow@google.com>
Found-by: 3pvd <3pvd@google.com>
Reviewed-by: Andy Nguyen <theflow@google.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

static int mov_read_coll(MOVContext *c, AVIOContext *pb, MOVAtom atom)
{
    MOVStreamContext *sc;
    int version;

    if (c->fc->nb_streams < 1)
        return AVERROR_INVALIDDATA;

    sc = c->fc->streams[c->fc->nb_streams - 1]->priv_data;

    if (atom.size < 5) {
        av_log(c->fc, AV_LOG_ERROR, "Empty Content Light Level box\n");
        return AVERROR_INVALIDDATA;
    }

    version = avio_r8(pb);
    if (version) {
        av_log(c->fc, AV_LOG_WARNING, "Unsupported Content Light Level box version %d\n", version);
        return 0;
    }
    avio_skip(pb, 3); /* flags */

    if (sc->coll){
        av_log(c->fc, AV_LOG_WARNING, "Ignoring duplicate COLL\n");
        return 0;
    }

    sc->coll = av_content_light_metadata_alloc(&sc->coll_size);
    if (!sc->coll)
        return AVERROR(ENOMEM);

    sc->coll->MaxCLL  = avio_rb16(pb);
    sc->coll->MaxFALL = avio_rb16(pb);

    return 0;
}


static int mov_read_clli(MOVContext *c, AVIOContext *pb, MOVAtom atom)
{
    MOVStreamContext *sc;

    if (c->fc->nb_streams < 1)
        return AVERROR_INVALIDDATA;

    sc = c->fc->streams[c->fc->nb_streams - 1]->priv_data;

    if (atom.size < 4) {
        av_log(c->fc, AV_LOG_ERROR, "Empty Content Light Level Info box\n");
        return AVERROR_INVALIDDATA;
    }

    if (sc->coll){
        av_log(c->fc, AV_LOG_WARNING, "Ignoring duplicate CLLI/COLL\n");
        return 0;
    }

    sc->coll = av_content_light_metadata_alloc(&sc->coll_size);
    if (!sc->coll)
        return AVERROR(ENOMEM);

    sc->coll->MaxCLL  = avio_rb16(pb);
    sc->coll->MaxFALL = avio_rb16(pb);

    return 0;
}


static int mov_read_default(MOVContext *c, AVIOContext *pb, MOVAtom atom)
{
    int64_t total_size = 0;
    MOVAtom a;
    int i;

    if (c->atom_depth > 10) {
        av_log(c->fc, AV_LOG_ERROR, "Atoms too deeply nested\n");
        return AVERROR_INVALIDDATA;
    }
    c->atom_depth ++;

    if (atom.size < 0)
        atom.size = INT64_MAX;
    while (total_size <= atom.size - 8) {
        int (*parse)(MOVContext*, AVIOContext*, MOVAtom) = NULL;
        a.size = avio_rb32(pb);
        a.type = avio_rl32(pb);
        if (avio_feof(pb))
            break;
        if (((a.type == MKTAG('f','r','e','e') && c->moov_retry) ||
              a.type == MKTAG('h','o','o','v')) &&
            a.size >= 8 &&
            c->fc->strict_std_compliance < FF_COMPLIANCE_STRICT) {
                uint32_t type;
                avio_skip(pb, 4);
                type = avio_rl32(pb);
                if (avio_feof(pb))
                    break;
                avio_seek(pb, -8, SEEK_CUR);
                if (type == MKTAG('m','v','h','d') ||
                    type == MKTAG('c','m','o','v')) {
                    av_log(c->fc, AV_LOG_ERROR, "Detected moov in a free or hoov atom.\n");
                    a.type = MKTAG('m','o','o','v');
                }
        }
        if (atom.type != MKTAG('r','o','o','t') &&
            atom.type != MKTAG('m','o','o','v')) {
            if (a.type == MKTAG('t','r','a','k') ||
                a.type == MKTAG('m','d','a','t')) {
                av_log(c->fc, AV_LOG_ERROR, "Broken file, trak/mdat not at top-level\n");
                avio_skip(pb, -8);
                c->atom_depth --;
                return 0;
            }
        }
        total_size += 8;
        if (a.size == 1 && total_size + 8 <= atom.size) { /* 64 bit extended size */
            a.size = avio_rb64(pb) - 8;
            total_size += 8;
        }
        av_log(c->fc, AV_LOG_TRACE, "type:'%s' parent:'%s' sz: %"PRId64" %"PRId64" %"PRId64"\n",
               av_fourcc2str(a.type), av_fourcc2str(atom.type), a.size, total_size, atom.size);
        if (a.size == 0) {
            a.size = atom.size - total_size + 8;
        }
        if (a.size < 0)
            break;
        a.size -= 8;
        if (a.size < 0)
            break;
        a.size = FFMIN(a.size, atom.size - total_size);

        for (i = 0; mov_default_parse_table[i].type; i++)
            if (mov_default_parse_table[i].type == a.type) {
                parse = mov_default_parse_table[i].parse;
                break;
            }

        // container is user data
        if (!parse && (atom.type == MKTAG('u','d','t','a') ||
                       atom.type == MKTAG('i','l','s','t')))
            parse = mov_read_udta_string;

        // Supports parsing the QuickTime Metadata Keys.
        // https://developer.apple.com/library/mac/documentation/QuickTime/QTFF/Metadata/Metadata.html
        if (!parse && c->found_hdlr_mdta &&
            atom.type == MKTAG('m','e','t','a') &&
            a.type == MKTAG('k','e','y','s') &&
            c->meta_keys_count == 0) {
            parse = mov_read_keys;
        }

        if (!parse) { /* skip leaf atoms data */
            avio_skip(pb, a.size);
        } else {
            int64_t start_pos = avio_tell(pb);
            int64_t left;
            int err = parse(c, pb, a);
            if (err < 0) {
                c->atom_depth --;
                return err;
            }
            if (c->found_moov && c->found_mdat && a.size <= INT64_MAX - start_pos &&
                ((!(pb->seekable & AVIO_SEEKABLE_NORMAL) || c->fc->flags & AVFMT_FLAG_IGNIDX || c->frag_index.complete) ||
                 start_pos + a.size == avio_size(pb))) {
                if (!(pb->seekable & AVIO_SEEKABLE_NORMAL) || c->fc->flags & AVFMT_FLAG_IGNIDX || c->frag_index.complete)
                    c->next_root_atom = start_pos + a.size;
                c->atom_depth --;
                return 0;
            }
            left = a.size - avio_tell(pb) + start_pos;
            if (left > 0) /* skip garbage at atom end */
                avio_skip(pb, left);
            else if (left < 0) {
                av_log(c->fc, AV_LOG_WARNING,
                       "overread end of atom '%s' by %"PRId64" bytes\n",
                       av_fourcc2str(a.type), -left);
                avio_seek(pb, left, SEEK_CUR);
            }
        }

        total_size += a.size;
    }

    if (total_size < atom.size && atom.size < 0x7ffff)
        avio_skip(pb, atom.size - total_size);

    c->atom_depth --;
    return 0;
}


static int mov_read_packet(AVFormatContext *s, AVPacket *pkt)
{
    MOVContext *mov = s->priv_data;
    MOVStreamContext *sc;
    AVIndexEntry *sample;
    AVStream *st = NULL;
    int64_t current_index;
    int ret;
    mov->fc = s;
 retry:
    sample = mov_find_next_sample(s, &st);
    if (!sample || (mov->next_root_atom && sample->pos > mov->next_root_atom)) {
        if (!mov->next_root_atom)
            return AVERROR_EOF;
        if ((ret = mov_switch_root(s, mov->next_root_atom, -1)) < 0)
            return ret;
        goto retry;
    }
    sc = st->priv_data;
    /* must be done just before reading, to avoid infinite loop on sample */
    current_index = sc->current_index;
    mov_current_sample_inc(sc);

    if (mov->next_root_atom) {
        sample->pos = FFMIN(sample->pos, mov->next_root_atom);
        sample->size = FFMIN(sample->size, (mov->next_root_atom - sample->pos));
    }

    if (st->discard != AVDISCARD_ALL) {
        int64_t ret64 = avio_seek(sc->pb, sample->pos, SEEK_SET);
        if (ret64 != sample->pos) {
            av_log(mov->fc, AV_LOG_ERROR, "stream %d, offset 0x%"PRIx64": partial file\n",
                   sc->ffindex, sample->pos);
            if (should_retry(sc->pb, ret64)) {
                mov_current_sample_dec(sc);
            } else if (ret64 < 0) {
                return (int)ret64;
            }
            return AVERROR_INVALIDDATA;
        }

        if (st->discard == AVDISCARD_NONKEY && !(sample->flags & AVINDEX_KEYFRAME)) {
            av_log(mov->fc, AV_LOG_DEBUG, "Nonkey frame from stream %d discarded due to AVDISCARD_NONKEY\n", sc->ffindex);
            goto retry;
        }

        if (st->codecpar->codec_id == AV_CODEC_ID_EIA_608 && sample->size > 8)
            ret = get_eia608_packet(sc->pb, pkt, sample->size);
        else
            ret = av_get_packet(sc->pb, pkt, sample->size);
        if (ret < 0) {
            if (should_retry(sc->pb, ret)) {
                mov_current_sample_dec(sc);
            }
            return ret;
        }
#if CONFIG_DV_DEMUXER
        if (mov->dv_demux && sc->dv_audio_container) {
            ret = avpriv_dv_produce_packet(mov->dv_demux, pkt, pkt->data, pkt->size, pkt->pos);
            av_packet_unref(pkt);
            if (ret < 0)
                return ret;
            ret = avpriv_dv_get_packet(mov->dv_demux, pkt);
            if (ret < 0)
                return ret;
        }
#endif
        if (sc->has_palette) {
            uint8_t *pal;

            pal = av_packet_new_side_data(pkt, AV_PKT_DATA_PALETTE, AVPALETTE_SIZE);
            if (!pal) {
                av_log(mov->fc, AV_LOG_ERROR, "Cannot append palette to packet\n");
            } else {
                memcpy(pal, sc->palette, AVPALETTE_SIZE);
                sc->has_palette = 0;
            }
        }
        if (st->codecpar->codec_id == AV_CODEC_ID_MP3 && !ffstream(st)->need_parsing && pkt->size > 4) {
            if (ff_mpa_check_header(AV_RB32(pkt->data)) < 0)
                ffstream(st)->need_parsing = AVSTREAM_PARSE_FULL;
        }
    }

    pkt->stream_index = sc->ffindex;
    pkt->dts = sample->timestamp;
    if (sample->flags & AVINDEX_DISCARD_FRAME) {
        pkt->flags |= AV_PKT_FLAG_DISCARD;
    }
    if (sc->ctts_data && sc->ctts_index < sc->ctts_count) {
        pkt->pts = pkt->dts + sc->dts_shift + sc->ctts_data[sc->ctts_index].duration;
        /* update ctts context */
        sc->ctts_sample++;
        if (sc->ctts_index < sc->ctts_count &&
            sc->ctts_data[sc->ctts_index].count == sc->ctts_sample) {
            sc->ctts_index++;
            sc->ctts_sample = 0;
        }
    } else {
        int64_t next_dts = (sc->current_sample < ffstream(st)->nb_index_entries) ?
            ffstream(st)->index_entries[sc->current_sample].timestamp : st->duration;

        if (next_dts >= pkt->dts)
            pkt->duration = next_dts - pkt->dts;
        pkt->pts = pkt->dts;
    }
    if (st->discard == AVDISCARD_ALL)
        goto retry;
    if (sc->sdtp_data && sc->current_sample <= sc->sdtp_count) {
        uint8_t sample_flags = sc->sdtp_data[sc->current_sample - 1];
        uint8_t sample_is_depended_on = (sample_flags >> 2) & 0x3;
        pkt->flags |= sample_is_depended_on == MOV_SAMPLE_DEPENDENCY_NO ? AV_PKT_FLAG_DISPOSABLE : 0;
    }
    pkt->flags |= sample->flags & AVINDEX_KEYFRAME ? AV_PKT_FLAG_KEY : 0;
    pkt->pos = sample->pos;

    /* Multiple stsd handling. */
    if (sc->stsc_data) {
        if (sc->stsc_data[sc->stsc_index].id > 0 &&
            sc->stsc_data[sc->stsc_index].id - 1 < sc->stsd_count &&
            sc->stsc_data[sc->stsc_index].id - 1 != sc->last_stsd_index) {
            ret = mov_change_extradata(sc, pkt);
            if (ret < 0)
                return ret;
        }

        /* Update the stsc index for the next sample */
        sc->stsc_sample++;
        if (mov_stsc_index_valid(sc->stsc_index, sc->stsc_count) &&
            mov_get_stsc_samples(sc, sc->stsc_index) == sc->stsc_sample) {
            sc->stsc_index++;
            sc->stsc_sample = 0;
        }
    }

    if (mov->aax_mode)
        aax_filter(pkt->data, pkt->size, mov);

    ret = cenc_filter(mov, st, sc, pkt, current_index);
    if (ret < 0) {
        return ret;
    }

    return 0;
}


static int mov_read_mvhd(MOVContext *c, AVIOContext *pb, MOVAtom atom)
{
    int i;
    int64_t creation_time;
    int version = avio_r8(pb); /* version */
    avio_rb24(pb); /* flags */

    if (version == 1) {
        creation_time = avio_rb64(pb);
        avio_rb64(pb);
    } else {
        creation_time = avio_rb32(pb);
        avio_rb32(pb); /* modification time */
    }
    mov_metadata_creation_time(&c->fc->metadata, creation_time, c->fc);
    c->time_scale = avio_rb32(pb); /* time scale */
    if (c->time_scale <= 0) {
        av_log(c->fc, AV_LOG_ERROR, "Invalid mvhd time scale %d, defaulting to 1\n", c->time_scale);
        c->time_scale = 1;
    }
    av_log(c->fc, AV_LOG_TRACE, "time scale = %i\n", c->time_scale);

    c->duration = (version == 1) ? avio_rb64(pb) : avio_rb32(pb); /* duration */
    // set the AVFormatContext duration because the duration of individual tracks
    // may be inaccurate
    if (!c->trex_data)
        c->fc->duration = av_rescale(c->duration, AV_TIME_BASE, c->time_scale);
    avio_rb32(pb); /* preferred scale */

    avio_rb16(pb); /* preferred volume */

    avio_skip(pb, 10); /* reserved */

    /* movie display matrix, store it in main context and use it later on */
    for (i = 0; i < 3; i++) {
        c->movie_display_matrix[i][0] = avio_rb32(pb); // 16.16 fixed point
        c->movie_display_matrix[i][1] = avio_rb32(pb); // 16.16 fixed point
        c->movie_display_matrix[i][2] = avio_rb32(pb); //  2.30 fixed point
    }

    avio_rb32(pb); /* preview time */
    avio_rb32(pb); /* preview duration */
    avio_rb32(pb); /* poster time */
    avio_rb32(pb); /* selection time */
    avio_rb32(pb); /* selection duration */
    avio_rb32(pb); /* current time */
    avio_rb32(pb); /* next track ID */

    return 0;
}


static int mov_read_tfdt(MOVContext *c, AVIOContext *pb, MOVAtom atom)
{
    MOVFragment *frag = &c->fragment;
    AVStream *st = NULL;
    MOVStreamContext *sc;
    int version, i;
    MOVFragmentStreamInfo * frag_stream_info;
    int64_t base_media_decode_time;

    for (i = 0; i < c->fc->nb_streams; i++) {
        if (c->fc->streams[i]->id == frag->track_id) {
            st = c->fc->streams[i];
            break;
        }
    }
    if (!st) {
        av_log(c->fc, AV_LOG_WARNING, "could not find corresponding track id %u\n", frag->track_id);
        return 0;
    }
    sc = st->priv_data;
    if (sc->pseudo_stream_id + 1 != frag->stsd_id && sc->pseudo_stream_id != -1)
        return 0;
    version = avio_r8(pb);
    avio_rb24(pb); /* flags */
    if (version) {
        base_media_decode_time = avio_rb64(pb);
    } else {
        base_media_decode_time = avio_rb32(pb);
    }

    frag_stream_info = get_current_frag_stream_info(&c->frag_index);
    if (frag_stream_info)
        frag_stream_info->tfdt_dts = base_media_decode_time;
    sc->track_end = base_media_decode_time;

    return 0;
}


static int mov_skip_multiple_stsd(MOVContext *c, AVIOContext *pb,
                                  int codec_tag, int format,
                                  int64_t size)
{
    if (codec_tag &&
         (codec_tag != format &&
          // AVID 1:1 samples with differing data format and codec tag exist
          (codec_tag != AV_RL32("AV1x") || format != AV_RL32("AVup")) &&
          // prores is allowed to have differing data format and codec tag
          codec_tag != AV_RL32("apcn") && codec_tag != AV_RL32("apch") &&
          // so is dv (sigh)
          codec_tag != AV_RL32("dvpp") && codec_tag != AV_RL32("dvcp") &&
          (c->fc->video_codec_id ? ff_codec_get_id(ff_codec_movvideo_tags, format) != c->fc->video_codec_id
                                 : codec_tag != MKTAG('j','p','e','g')))) {
        /* Multiple fourcc, we skip JPEG. This is not correct, we should
         * export it as a separate AVStream but this needs a few changes
         * in the MOV demuxer, patch welcome. */

        av_log(c->fc, AV_LOG_WARNING, "multiple fourcc not supported\n");
        avio_skip(pb, size);
        return 1;
    }

    return 0;
}