details: http://hg.nginx.org/nginx/rev/a08fad30aeac
branches:
changeset: 6050:a08fad30aeac
user: Maxim Dounin <mdounin@mdounin.ru>
date: Mon Mar 23 21:09:19 2015 +0300
description:
Request body: unbuffered reading.

The r->request_body_no_buffering flag was introduced. It instructs
client request body reading code to avoid reading the whole body, and
to call post_handler early instead. The caller should use the
ngx_http_read_unbuffered_request_body() function to read remaining
parts of the body.

Upstream module is now able to use this mode, if configured with
the proxy_request_buffering directive.

diffstat:

src/http/modules/ngx_http_proxy_module.c | 26 ++++-
src/http/ngx_http.h | 1 +
src/http/ngx_http_request.c | 5 +
src/http/ngx_http_request.h | 2 +
src/http/ngx_http_request_body.c | 107 ++++++++++++++++-
src/http/ngx_http_upstream.c | 185 ++++++++++++++++++++++++++++--
src/http/ngx_http_upstream.h | 1 +
src/http/ngx_http_variables.c | 4 +
8 files changed, 306 insertions(+), 25 deletions(-)

diffs (truncated from 602 to 300 lines):

diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -292,6 +292,13 @@ static ngx_command_t ngx_http_proxy_com
offsetof(ngx_http_proxy_loc_conf_t, upstream.buffering),
NULL },

+ { ngx_string("proxy_request_buffering"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_proxy_loc_conf_t, upstream.request_buffering),
+ NULL },
+
{ ngx_string("proxy_ignore_client_abort"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
ngx_conf_set_flag_slot,
@@ -876,6 +883,15 @@ ngx_http_proxy_handler(ngx_http_request_

u->accel = 1;

+ if (!plcf->upstream.request_buffering
+ && plcf->body_values == NULL && plcf->upstream.pass_request_body
+ && !r->headers_in.chunked)
+ {
+ /* TODO: support chunked when using HTTP/1.1 */
+
+ r->request_body_no_buffering = 1;
+ }
+
rc = ngx_http_read_client_request_body(r, ngx_http_upstream_init);

if (rc >= NGX_HTTP_SPECIAL_RESPONSE) {
@@ -1393,7 +1409,11 @@ ngx_http_proxy_create_request(ngx_http_r
"http proxy header:%N\"%*s\"",
(size_t) (b->last - b->pos), b->pos);

- if (plcf->body_values == NULL && plcf->upstream.pass_request_body) {
+ if (r->request_body_no_buffering) {
+
+ u->request_bufs = cl;
+
+ } else if (plcf->body_values == NULL && plcf->upstream.pass_request_body) {

body = u->request_bufs;
u->request_bufs = cl;
@@ -2582,6 +2602,7 @@ ngx_http_proxy_create_loc_conf(ngx_conf_
conf->upstream.store_access = NGX_CONF_UNSET_UINT;
conf->upstream.next_upstream_tries = NGX_CONF_UNSET_UINT;
conf->upstream.buffering = NGX_CONF_UNSET;
+ conf->upstream.request_buffering = NGX_CONF_UNSET;
conf->upstream.ignore_client_abort = NGX_CONF_UNSET;
conf->upstream.force_ranges = NGX_CONF_UNSET;

@@ -2691,6 +2712,9 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t
ngx_conf_merge_value(conf->upstream.buffering,
prev->upstream.buffering, 1);

+ ngx_conf_merge_value(conf->upstream.request_buffering,
+ prev->upstream.request_buffering, 1);
+
ngx_conf_merge_value(conf->upstream.ignore_client_abort,
prev->upstream.ignore_client_abort, 0);

diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h
--- a/src/http/ngx_http.h
+++ b/src/http/ngx_http.h
@@ -138,6 +138,7 @@ ngx_int_t ngx_http_send_special(ngx_http

ngx_int_t ngx_http_read_client_request_body(ngx_http_request_t *r,
ngx_http_client_body_handler_pt post_handler);
+ngx_int_t ngx_http_read_unbuffered_request_body(ngx_http_request_t *r);

ngx_int_t ngx_http_send_header(ngx_http_request_t *r);
ngx_int_t ngx_http_special_response_handler(ngx_http_request_t *r,
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -2525,6 +2525,11 @@ ngx_http_finalize_connection(ngx_http_re
return;
}

+ if (r->reading_body) {
+ r->keepalive = 0;
+ r->lingering_close = 1;
+ }
+
if (!ngx_terminate
&& !ngx_exiting
&& r->keepalive
diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h
--- a/src/http/ngx_http_request.h
+++ b/src/http/ngx_http_request.h
@@ -473,6 +473,7 @@ struct ngx_http_request_s {
unsigned request_body_in_clean_file:1;
unsigned request_body_file_group_access:1;
unsigned request_body_file_log_level:3;
+ unsigned request_body_no_buffering:1;

unsigned subrequest_in_memory:1;
unsigned waited:1;
@@ -509,6 +510,7 @@ struct ngx_http_request_s {
unsigned keepalive:1;
unsigned lingering_close:1;
unsigned discard_body:1;
+ unsigned reading_body:1;
unsigned internal:1;
unsigned error_page:1;
unsigned filter_finalize:1;
diff --git a/src/http/ngx_http_request_body.c b/src/http/ngx_http_request_body.c
--- a/src/http/ngx_http_request_body.c
+++ b/src/http/ngx_http_request_body.c
@@ -42,12 +42,14 @@ ngx_http_read_client_request_body(ngx_ht

#if (NGX_HTTP_SPDY)
if (r->spdy_stream && r == r->main) {
+ r->request_body_no_buffering = 0;
rc = ngx_http_spdy_read_request_body(r, post_handler);
goto done;
}
#endif

if (r != r->main || r->request_body || r->discard_body) {
+ r->request_body_no_buffering = 0;
post_handler(r);
return NGX_OK;
}
@@ -57,6 +59,10 @@ ngx_http_read_client_request_body(ngx_ht
goto done;
}

+ if (r->request_body_no_buffering) {
+ r->request_body_in_file_only = 0;
+ }
+
rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t));
if (rb == NULL) {
rc = NGX_HTTP_INTERNAL_SERVER_ERROR;
@@ -79,6 +85,7 @@ ngx_http_read_client_request_body(ngx_ht
r->request_body = rb;

if (r->headers_in.content_length_n < 0 && !r->headers_in.chunked) {
+ r->request_body_no_buffering = 0;
post_handler(r);
return NGX_OK;
}
@@ -171,6 +178,8 @@ ngx_http_read_client_request_body(ngx_ht
}
}

+ r->request_body_no_buffering = 0;
+
post_handler(r);

return NGX_OK;
@@ -214,6 +223,21 @@ ngx_http_read_client_request_body(ngx_ht

done:

+ if (r->request_body_no_buffering
+ && (rc == NGX_OK || rc == NGX_AGAIN))
+ {
+ if (rc == NGX_OK) {
+ r->request_body_no_buffering = 0;
+
+ } else {
+ /* rc == NGX_AGAIN */
+ r->reading_body = 1;
+ }
+
+ r->read_event_handler = ngx_http_block_reading;
+ post_handler(r);
+ }
+
if (rc >= NGX_HTTP_SPECIAL_RESPONSE) {
r->main->count--;
}
@@ -222,6 +246,26 @@ done:
}


+ngx_int_t
+ngx_http_read_unbuffered_request_body(ngx_http_request_t *r)
+{
+ ngx_int_t rc;
+
+ if (r->connection->read->timedout) {
+ r->connection->timedout = 1;
+ return NGX_HTTP_REQUEST_TIME_OUT;
+ }
+
+ rc = ngx_http_do_read_client_request_body(r);
+
+ if (rc == NGX_OK) {
+ r->reading_body = 0;
+ }
+
+ return rc;
+}
+
+
static void
ngx_http_read_client_request_body_handler(ngx_http_request_t *r)
{
@@ -264,18 +308,43 @@ ngx_http_do_read_client_request_body(ngx
for ( ;; ) {
if (rb->buf->last == rb->buf->end) {

- /* pass buffer to request body filter chain */
+ if (rb->buf->pos != rb->buf->last) {

- out.buf = rb->buf;
- out.next = NULL;
+ /* pass buffer to request body filter chain */

- rc = ngx_http_request_body_filter(r, &out);
+ out.buf = rb->buf;
+ out.next = NULL;

- if (rc != NGX_OK) {
- return rc;
+ rc = ngx_http_request_body_filter(r, &out);
+
+ if (rc != NGX_OK) {
+ return rc;
+ }
+
+ } else {
+
+ /* update chains */
+
+ rc = ngx_http_request_body_filter(r, NULL);
+
+ if (rc != NGX_OK) {
+ return rc;
+ }
}

if (rb->busy != NULL) {
+ if (r->request_body_no_buffering) {
+ if (c->read->timer_set) {
+ ngx_del_timer(c->read);
+ }
+
+ if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+ return NGX_HTTP_INTERNAL_SERVER_ERROR;
+ }
+
+ return NGX_AGAIN;
+ }
+
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}

@@ -342,6 +411,22 @@ ngx_http_do_read_client_request_body(ngx
}

if (!c->read->ready) {
+
+ if (r->request_body_no_buffering
+ && rb->buf->pos != rb->buf->last)
+ {
+ /* pass buffer to request body filter chain */
+
+ out.buf = rb->buf;
+ out.next = NULL;
+
+ rc = ngx_http_request_body_filter(r, &out);
+
+ if (rc != NGX_OK) {
+ return rc;
+ }
+ }
+
clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
ngx_add_timer(c->read, clcf->client_body_timeout);

@@ -387,9 +472,10 @@ ngx_http_do_read_client_request_body(ngx
}
}

- r->read_event_handler = ngx_http_block_reading;
-
- rb->post_handler(r);
+ if (!r->request_body_no_buffering) {
+ r->read_event_handler = ngx_http_block_reading;
+ rb->post_handler(r);
+ }

return NGX_OK;
}
@@ -1085,7 +1171,8 @@ ngx_http_request_body_save_filter(ngx_ht
}

if (rb->rest > 0
- && rb->buf && rb->buf->last == rb->buf->end)
+ && rb->buf && rb->buf->last == rb->buf->end
+ && !r->request_body_no_buffering)
{
if (ngx_http_write_request_body(r) != NGX_OK) {

_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel