Welcome! Log In Create A New Profile

Advanced

Re: upstream keepalive - call for testing

All files from this thread

File Name File Size   Posted by Date  
upstream.patch 2.3 KB open | download Matthieu Tourne 08/12/2011 Read message
ngx_http_upstream_keepalive.patch 542 bytes open | download Matthieu Tourne 08/12/2011 Read message
upstream.patch 2.6 KB open | download Matthieu Tourne 08/12/2011 Read message
September 15, 2011 01:52PM
Hello,
I have wrote a module to make nginx support 304 to decrease bandwidth usage..
note: I have a newbie for nginx module development, so the above module may
have some problem. Welcome to test it and feedback another problem with me.

You can download full patch file from here:
http://m-b.cc/share/proxy_304.txt

# User MagicBear <magicbearmo@gmail.com>
Upstream:
add $upstream_last_modified variant.
add handler for 304 Unmodified.
Proxy:
change to send If-Modified-Since header.

TODO:
change write TO not block IO.

diff -ruN a/http/modules/ngx_http_proxy_module.c
b/http/modules/ngx_http_proxy_module.c
--- a/http/modules/ngx_http_proxy_module.c 2011-09-15
22:23:03.284431407 +0800
+++ b/http/modules/ngx_http_proxy_module.c 2011-09-16
01:41:44.654428632 +0800
@@ -543,7 +543,7 @@
{ ngx_string("Connection"), ngx_string("close") },
{ ngx_string("Keep-Alive"), ngx_string("") },
{ ngx_string("Expect"), ngx_string("") },
- { ngx_string("If-Modified-Since"), ngx_string("") },
+ { ngx_string("If-Modified-Since"),
ngx_string("$upstream_last_modified") },
{ ngx_string("If-Unmodified-Since"), ngx_string("") },
{ ngx_string("If-None-Match"), ngx_string("") },
{ ngx_string("If-Match"), ngx_string("") },
diff -ruN a/http/ngx_http_upstream.c b/http/ngx_http_upstream.c
--- a/http/ngx_http_upstream.c 2011-09-15 22:23:03.284431407 +0800
+++ b/http/ngx_http_upstream.c 2011-09-16 01:41:44.654428632 +0800
@@ -16,6 +16,8 @@
ngx_http_upstream_t *u);
static ngx_int_t ngx_http_upstream_cache_status(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_http_upstream_last_modified(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data);
#endif

static void ngx_http_upstream_init_request(ngx_http_request_t *r);
@@ -342,6 +344,10 @@
ngx_http_upstream_cache_status, 0,
NGX_HTTP_VAR_NOCACHEABLE, 0 },

+ { ngx_string("upstream_last_modified"), NULL,
+ ngx_http_upstream_last_modified, 0,
+ NGX_HTTP_VAR_NOCACHEABLE, 0 },
+
#endif

{ ngx_null_string, NULL, NULL, 0, 0, 0 }
@@ -1618,6 +1624,80 @@
u->buffer.last = u->buffer.pos;
}

+#if (NGX_HTTP_CACHE)
+
+ if (u->cache_status == NGX_HTTP_CACHE_EXPIRED &&
+ u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED &&
+ ngx_http_file_cache_valid(u->conf->cache_valid,
u->headers_in.status_n))
+ {
+ ngx_int_t rc;
+
+ rc = u->reinit_request(r);
+
+ if (rc == NGX_OK) {
+ u->cache_status = NGX_HTTP_CACHE_BYPASS;
+ rc = ngx_http_upstream_cache_send(r, u);
+
+ time_t now, valid;
+
+ now = ngx_time();
+
+ valid = r->cache->valid_sec;
+
+ if (valid == 0) {
+ valid =
ngx_http_file_cache_valid(u->conf->cache_valid,
+
u->headers_in.status_n);
+ if (valid) {
+ r->cache->valid_sec = now +
valid;
+ }
+ }
+
+ if (valid) {
+ r->cache->last_modified =
r->headers_out.last_modified_time;
+ r->cache->date = now;
+ r->cache->body_start = (u_short)
(u->buffer.pos - u->buffer.start);
+
+ // update Header
+ ngx_http_file_cache_set_header(r,
u->buffer.start);
+
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP,
r->connection->log, 0,
+
"update cache \"%s\" header to new expired." , r->cache->file.name.data);
+
+ // Reopen file via RW
+ ngx_fd_t fd =
ngx_open_file(r->cache->file.name.data, NGX_FILE_RDWR, NGX_FILE_OPEN, 0);
+
+ if (fd == NGX_INVALID_FILE) {
+ ngx_log_error(NGX_LOG_CRIT,
r->connection->log, ngx_errno,
+
ngx_open_file_n " \"%s\" failed", r->cache->file.name.data);
+ return;
+ }
+
+ // Write cache
+ if (write(fd, u->buffer.start,
sizeof(ngx_http_file_cache_header_t)) < 0)
+ {
+ ngx_log_error(NGX_LOG_CRIT,
r->connection->log, ngx_errno,
+
"write proxy_cache \"%s\" failed", r->cache->file.name.data);
+ return;
+ }
+
+ if (ngx_close_file(fd) ==
NGX_FILE_ERROR) {
+ ngx_log_error(NGX_LOG_ALERT,
r->connection->log, ngx_errno,
+
ngx_close_file_n " \"%s\" failed", r->cache->file.name.data);
+ }
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP,
r->connection->log, 0,
+
"update cache \"%s\" header to new expired done." ,
r->cache->file.name.data);
+ } else {
+ u->cacheable = 0;
+ r->headers_out.last_modified_time =
-1;
+ }
+ }
+
+ ngx_http_upstream_finalize_request(r, u, rc);
+ return;
+ }
+
+#endif
+
if (ngx_http_upstream_test_next(r, u) == NGX_OK) {
return;
}
@@ -4006,6 +4086,32 @@

return NGX_OK;
}
+
+ngx_int_t
+ngx_http_upstream_last_modified(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data)
+{
+ u_char *u;
+
+ if (r->upstream == NULL || r->upstream->cache_status == 0 ||
r->cache==NULL || r->cache->last_modified <= 0) {
+ v->not_found = 1;
+ return NGX_OK;
+ }
+
+ v->valid = 1;
+ v->no_cacheable = 0;
+ v->not_found = 0;
+ u = ngx_pcalloc(r->pool, 30);
+ if (u == NULL) {
+ return NGX_ERROR;
+ }
+
+ v->len = 29;
+ ngx_http_time(u, r->cache->last_modified);
+ v->data = u;
+
+ return NGX_OK;
+}

#endif


MagicBear

2011/9/15 magicbear <nginx-forum@nginx.us>

> I have run the nginx 1.1.2 via this patch for 7 days, except for one
> days have a large DDoS so I restart nginx for several seconds, it was
> very stable to work.
> Handle about 70million request without problem happen, I think the last
> problem may be have a memory corruption, you are right.
> I will check that server when have times.
> Thanks for your hard work.
>
> MagicBear
>
> Maxim Dounin Wrote:
> -------------------------------------------------------
> > Hello!
> >
> > On Mon, Sep 05, 2011 at 11:42:31PM +0800,
> > ビリビリⅤ wrote:
> >
> > > (gdb) fr 0
> > > #0 ngx_http_upstream_handler
> > (ev=0x7fc45735f8a8)
> > > at src/http/ngx_http_upstream.c:915
> > > 915 ctx->current_request = r;
> > > (gdb) p ngx_cycle->log
> > > $1 = (ngx_log_t *) 0x21f19a8
> > > (gdb) p *r
> > > $2 = {signature = 51686928, connection =
> > 0x23b4160, ctx = 0x0,
> >
> > [...]
> >
> > This looks like memory corruption, but
> > unfortunately I don't see
> > any traces of the real cause. My best quess is
> > improper handling
> > of proxy_ignore_client_abort as fixed in 1.1.2.
> > Please try 1.1.2
> > with patches from
> >
> > http://nginx.org/patches/patch-nginx-keepalive-ful
> > l-5.txt
> >
> > It already includes upstream keepalive module, as
> > well as all
> > other upstream-keepalive related fixes. See here
> > for details:
> >
> > http://mailman.nginx.org/pipermail/nginx-devel/201
> > 1-September/001147.html
> >
> > Maxim Dounin
> >
> > _______________________________________________
> > nginx mailing list
> > nginx@nginx.org
> > http://mailman.nginx.org/mailman/listinfo/nginx
>
> Posted at Nginx Forum:
> http://forum.nginx.org/read.php?2,213207,215217#msg-215217
>
> _______________________________________________
> nginx mailing list
> nginx@nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx
>
# User MagicBear <magicbearmo@gmail.com>
Upstream:
add $upstream_last_modified variant.
add handler for 304 Unmodified.
Proxy:
change to send If-Modified-Since header.

TODO:
change write TO not block IO.

diff -ruN a/http/modules/ngx_http_proxy_module.c b/http/modules/ngx_http_proxy_module.c
--- a/http/modules/ngx_http_proxy_module.c 2011-09-15 22:23:03.284431407 +0800
+++ b/http/modules/ngx_http_proxy_module.c 2011-09-16 01:41:44.654428632 +0800
@@ -543,7 +543,7 @@
{ ngx_string("Connection"), ngx_string("close") },
{ ngx_string("Keep-Alive"), ngx_string("") },
{ ngx_string("Expect"), ngx_string("") },
- { ngx_string("If-Modified-Since"), ngx_string("") },
+ { ngx_string("If-Modified-Since"), ngx_string("$upstream_last_modified") },
{ ngx_string("If-Unmodified-Since"), ngx_string("") },
{ ngx_string("If-None-Match"), ngx_string("") },
{ ngx_string("If-Match"), ngx_string("") },
diff -ruN a/http/ngx_http_upstream.c b/http/ngx_http_upstream.c
--- a/http/ngx_http_upstream.c 2011-09-15 22:23:03.284431407 +0800
+++ b/http/ngx_http_upstream.c 2011-09-16 01:41:44.654428632 +0800
@@ -16,6 +16,8 @@
ngx_http_upstream_t *u);
static ngx_int_t ngx_http_upstream_cache_status(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_http_upstream_last_modified(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data);
#endif

static void ngx_http_upstream_init_request(ngx_http_request_t *r);
@@ -342,6 +344,10 @@
ngx_http_upstream_cache_status, 0,
NGX_HTTP_VAR_NOCACHEABLE, 0 },

+ { ngx_string("upstream_last_modified"), NULL,
+ ngx_http_upstream_last_modified, 0,
+ NGX_HTTP_VAR_NOCACHEABLE, 0 },
+
#endif

{ ngx_null_string, NULL, NULL, 0, 0, 0 }
@@ -1618,6 +1624,80 @@
u->buffer.last = u->buffer.pos;
}

+#if (NGX_HTTP_CACHE)
+
+ if (u->cache_status == NGX_HTTP_CACHE_EXPIRED &&
+ u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED &&
+ ngx_http_file_cache_valid(u->conf->cache_valid, u->headers_in.status_n))
+ {
+ ngx_int_t rc;
+
+ rc = u->reinit_request(r);
+
+ if (rc == NGX_OK) {
+ u->cache_status = NGX_HTTP_CACHE_BYPASS;
+ rc = ngx_http_upstream_cache_send(r, u);
+
+ time_t now, valid;
+
+ now = ngx_time();
+
+ valid = r->cache->valid_sec;
+
+ if (valid == 0) {
+ valid = ngx_http_file_cache_valid(u->conf->cache_valid,
+ u->headers_in.status_n);
+ if (valid) {
+ r->cache->valid_sec = now + valid;
+ }
+ }
+
+ if (valid) {
+ r->cache->last_modified = r->headers_out.last_modified_time;
+ r->cache->date = now;
+ r->cache->body_start = (u_short) (u->buffer.pos - u->buffer.start);
+
+ // update Header
+ ngx_http_file_cache_set_header(r, u->buffer.start);
+
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "update cache \"%s\" header to new expired." , r->cache->file.name.data);
+
+ // Reopen file via RW
+ ngx_fd_t fd = ngx_open_file(r->cache->file.name.data, NGX_FILE_RDWR, NGX_FILE_OPEN, 0);
+
+ if (fd == NGX_INVALID_FILE) {
+ ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
+ ngx_open_file_n " \"%s\" failed", r->cache->file.name.data);
+ return;
+ }
+
+ // Write cache
+ if (write(fd, u->buffer.start, sizeof(ngx_http_file_cache_header_t)) < 0)
+ {
+ ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
+ "write proxy_cache \"%s\" failed", r->cache->file.name.data);
+ return;
+ }
+
+ if (ngx_close_file(fd) == NGX_FILE_ERROR) {
+ ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno,
+ ngx_close_file_n " \"%s\" failed", r->cache->file.name.data);
+ }
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "update cache \"%s\" header to new expired done." , r->cache->file.name.data);
+ } else {
+ u->cacheable = 0;
+ r->headers_out.last_modified_time = -1;
+ }
+ }
+
+ ngx_http_upstream_finalize_request(r, u, rc);
+ return;
+ }
+
+#endif
+
if (ngx_http_upstream_test_next(r, u) == NGX_OK) {
return;
}
@@ -4006,6 +4086,32 @@

return NGX_OK;
}
+
+ngx_int_t
+ngx_http_upstream_last_modified(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data)
+{
+ u_char *u;
+
+ if (r->upstream == NULL || r->upstream->cache_status == 0 || r->cache==NULL || r->cache->last_modified <= 0) {
+ v->not_found = 1;
+ return NGX_OK;
+ }
+
+ v->valid = 1;
+ v->no_cacheable = 0;
+ v->not_found = 0;
+ u = ngx_pcalloc(r->pool, 30);
+ if (u == NULL) {
+ return NGX_ERROR;
+ }
+
+ v->len = 29;
+ ngx_http_time(u, r->cache->last_modified);
+ v->data = u;
+
+ return NGX_OK;
+}

#endif
_______________________________________________
nginx mailing list
nginx@nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx
Subject Author Posted

upstream keepalive - call for testing

Maxim Dounin August 01, 2011 12:10PM

Re: upstream keepalive - call for testing

liseen August 02, 2011 09:38AM

Re: upstream keepalive - call for testing

Maxim Dounin August 02, 2011 01:34PM

Re: upstream keepalive - call for testing

António P. P. Almeida August 02, 2011 11:28AM

Re: upstream keepalive - call for testing

Maxim Dounin August 02, 2011 01:38PM

Re: upstream keepalive - call for testing

David Yu August 02, 2011 01:44PM

Re: upstream keepalive - call for testing

Maxim Dounin August 02, 2011 01:52PM

Re: upstream keepalive - call for testing

David Yu August 02, 2011 01:54PM

Re: upstream keepalive - call for testing

Maxim Dounin August 02, 2011 02:48PM

Re: upstream keepalive - call for testing

David Yu August 02, 2011 03:10PM

Re: upstream keepalive - call for testing

liseen August 02, 2011 11:58PM

Re: upstream keepalive - call for testing

splitice August 03, 2011 01:22AM

Re: upstream keepalive - call for testing

Matthieu Tourne August 03, 2011 08:08PM

Re: upstream keepalive - call for testing

Maxim Dounin August 04, 2011 02:54AM

Re: upstream keepalive - call for testing

splitice August 08, 2011 12:46AM

Re: upstream keepalive - call for testing

Maxim Dounin August 08, 2011 05:24AM

Re: upstream keepalive - call for testing

splitice August 08, 2011 05:36AM

Re: upstream keepalive - call for testing

splitice August 08, 2011 05:38AM

Re: upstream keepalive - call for testing Attachments

Matthieu Tourne August 12, 2011 03:34PM

Re: upstream keepalive - call for testing

Maxim Dounin August 12, 2011 04:00PM

Re: upstream keepalive - call for testing

Matthieu Tourne August 12, 2011 05:14PM

Re: upstream keepalive - call for testing

Maxim Dounin August 12, 2011 06:28PM

Re: upstream keepalive - call for testing Attachments

Matthieu Tourne August 12, 2011 06:44PM

Re: upstream keepalive - call for testing

Matthieu Tourne August 16, 2011 07:32PM

Re: upstream keepalive - call for testing

Maxim Dounin August 16, 2011 08:24PM

Re: upstream keepalive - call for testing

magicbear August 24, 2011 01:11PM

Re: upstream keepalive - call for testing

Maxim Dounin August 24, 2011 08:06PM

Re: upstream keepalive - call for testing

sv August 24, 2011 09:18PM

Re: upstream keepalive - call for testing

magicbear August 25, 2011 01:30AM

Re: upstream keepalive - call for testing

magicbear August 26, 2011 03:08AM

Re: upstream keepalive - call for testing

Maxim Dounin August 26, 2011 05:40AM

Re: upstream keepalive - call for testing

magicbear August 26, 2011 07:01AM

Re: upstream keepalive - call for testing

magicbear August 26, 2011 07:04AM

Re: upstream keepalive - call for testing

magicbear August 26, 2011 07:28AM

Re: upstream keepalive - call for testing

Maxim Dounin August 26, 2011 07:38AM

upstream keepalive close connections actively

cfsego August 02, 2011 10:50PM

Re: upstream keepalive close connections actively

Maxim Dounin August 03, 2011 03:40AM

RE: upstream keepalive close connections actively

Charles Chen August 03, 2011 05:54AM

Re: upstream keepalive - call for testing

magicbear August 26, 2011 07:54AM

Re: upstream keepalive - call for testing

Maxim Dounin August 26, 2011 11:56AM

Re: upstream keepalive - call for testing

magicbear August 26, 2011 12:17PM

Re: upstream keepalive - call for testing

Maxim Dounin August 26, 2011 02:08PM

Re: upstream keepalive - call for testing

magicbear August 26, 2011 03:00PM

Re: upstream keepalive - call for testing

magicbear August 26, 2011 12:28PM

Re: upstream keepalive - call for testing

magicbear August 26, 2011 01:00PM

Re: upstream keepalive - call for testing

magicbear August 26, 2011 01:51PM

Re: upstream keepalive - call for testing

magicbear August 28, 2011 01:07PM

Re: upstream keepalive - call for testing

magicbear August 28, 2011 01:10PM

Re: upstream keepalive - call for testing

Maxim Dounin August 28, 2011 09:48PM

Re: upstream keepalive - call for testing

magicbear August 31, 2011 04:04PM

Re: upstream keepalive - call for testing

splitice August 31, 2011 09:58PM

Re: upstream keepalive - call for testing

magicbear September 01, 2011 09:38AM

Re: upstream keepalive - call for testing

magicbear September 04, 2011 01:33PM

Re: upstream keepalive - call for testing

Maxim Dounin September 04, 2011 02:22PM

Re: upstream keepalive - call for testing

magicbear September 04, 2011 02:34PM

Re: upstream keepalive - call for testing

Maxim Dounin September 05, 2011 03:10AM

Re: upstream keepalive - call for testing

ビリビリⅤ September 05, 2011 11:44AM

Re: upstream keepalive - call for testing

Maxim Dounin September 05, 2011 02:04PM

Re: upstream keepalive - call for testing

magicbear September 06, 2011 02:39AM

Re: upstream keepalive - call for testing

Matthieu Tourne September 07, 2011 07:36PM

Re: upstream keepalive - call for testing

Maxim Dounin September 08, 2011 05:28AM

Re: upstream keepalive - call for testing

Maxim Dounin September 08, 2011 11:44AM

Re: upstream keepalive - call for testing

Matthieu Tourne September 08, 2011 06:06PM

Re: upstream keepalive - call for testing

magicbear September 14, 2011 06:54PM

Re: upstream keepalive - call for testing

magicbear September 15, 2011 01:52PM

Re: upstream keepalive - call for testing

splitice September 15, 2011 09:44PM

Re: upstream keepalive - call for testing

philipp December 29, 2011 07:47AM

Re: upstream keepalive - call for testing

Maxim Dounin December 29, 2011 10:06AM

Re: upstream keepalive - call for testing

alexscott March 08, 2012 09:30AM

Re: upstream keepalive - call for testing

Andrew Alexeev March 09, 2012 01:20AM

Re: upstream keepalive - call for testing

alexscott March 12, 2012 10:35AM

Re: upstream keepalive - call for testing

Maxim Dounin March 12, 2012 10:56AM

Re: upstream keepalive - call for testing

alexscott March 12, 2012 01:40PM

Re: upstream keepalive - call for testing

alexscott March 12, 2012 03:55PM

Re: upstream keepalive - call for testing

Maxim Dounin March 12, 2012 02:00PM



Sorry, only registered users may post in this forum.

Click here to login

Online Users

Guests: 119
Record Number of Users: 6 on February 13, 2018
Record Number of Guests: 421 on December 02, 2018
Powered by nginx      Powered by FreeBSD      PHP Powered      Powered by MariaDB      ipv6 ready