Hi Maxim,

Thanks for your reply. Your guide made me understand thoroughly the role of calling ngx_http_next_body_filter(r, NULL) in the gzip module which helped a lot. The buffer now can be reused but I still got one issue that confused me a lot.

I got curl: (18) transfer closed with outstanding read data remaining error when I access the path the code modified. I captured packets through tcpdump and the last packet containing the response was marked as Malformed Packet.

Here's the code:

if (ctx->nomem) {
if (ngx_http_next_body_filter(r, NULL) == NGX_ERROR) {
goto failed;
}
ngx_chain_t *cl = NULL;
ngx_chain_update_chains(r->pool, &ctx->free, &ctx->busy, &cl,
(ngx_buf_tag_t) &ngx_http_my_filter_module);
ctx->nomem = 0;
flush = 0;
} else {
flush = ctx->busy ? 1 : 0;
}

for (;;) {

/* cycle while we can write to a client */

for (;;) {

/* cycle while there is data to insert into the beginning */

rc = ngx_http_my_get_buf(r, ctx);

if (rc == NGX_DECLINED) {
break;
}

if (rc == NGX_ERROR) {
goto failed;
}

/* there are buffers to write data */

// rc = operation to copy 64 kb data to the ctx->out_buf;
ctx->out_buf->last = ctx->out_buf->pos + 64 * 4096;

ngx_chain_t *cl = ngx_alloc_chain_link(r->pool);
if (cl == NULL) {
goto failed;
}

cl->buf = ctx->out_buf;
cl->next = NULL;

*ctx->last_out = cl;
ctx->last_out = &cl->next;

if (rc == OK) {
ctx->stage = DONE;
break;
}

/* rc == NGX_AGAIN */
}

if (ctx->out == NULL && !flush) {
return ctx->busy ? NGX_AGAIN : NGX_OK;
}

ngx_chain_t *a = ctx->out;
while (a) {
if (ngx_buf_size(a->buf)) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "beign: %*s, end: %*s",
10, a->buf->pos, 10, a->buf->last - 10); // add logging to make sure the buf is complete, all buf were logged
}
a = a->next;
}

rc = ngx_http_next_body_filter(r, ctx->out);

if (rc == NGX_ERROR) {
goto failed;
}

ngx_chain_update_chains(r->pool, &ctx->free, &ctx->busy, &ctx->out,
(ngx_buf_tag_t) &ngx_http_my_filter_module);
ctx->last_out = &ctx->out;

ctx->nomem = 0;
flush = 0;

if (ctx->stage == DONE) {
return rc;
}
}

static ngx_int_t ngx_http_my_get_buf(ngx_http_request_t *r, ngx_http_my_ctx_t *ctx) {

ngx_chain_t *cl;
ngx_http_my_loc_conf_t *conf = ngx_http_get_module_loc_conf(r, ngx_http_my_filter_module);

if (ctx->free) {

cl = ctx->free;
ctx->out_buf = cl->buf;
ctx->free = cl->next;

ngx_free_chain(r->pool, cl);

} else if (ctx->bufs < conf->bufs.num) {

ctx->out_buf = ngx_create_temp_buf(r->pool, conf->bufs.size);
if (ctx->out_buf == NULL) {
return NGX_ERROR;
}

ctx->out_buf->tag = (ngx_buf_tag_t) &ngx_http_my_filter_module;
ctx->out_buf->recycled = 1;
ctx->bufs++;

} else {
ctx->nomem = 1;
return NGX_DECLINED;
}

return NGX_OK;
}