Author: mdounin
Date: 2011-12-26 11:15:23 +0000 (Mon, 26 Dec 2011)
New Revision: 4386

Log:
Cache locks initial implementation.

New directives: proxy_cache_lock on/off, proxy_cache_lock_timeout. With
proxy_cache_lock set to on, only one request will be allowed to go to
upstream for a particular cache item. Others will wait for a response
to appear in cache (or cache lock released) up to proxy_cache_lock_timeout.

Waiting requests will recheck if they have cached response ready (or are
allowed to run) every 500ms.

Note: we intentionally don't intercept NGX_DECLINED possibly returned by
ngx_http_file_cache_read(). This needs more work (possibly safe, but needs
further investigation). Anyway, it's exceptional situation.

Note: probably there should be a way to disable caching of responses
if there is already one request fetching resource to cache (without waiting
at all). Two possible ways include another cache lock option ("no_cache")
or using proxy_no_cache with some supplied variable.

Note: probably there should be a way to lock updating requests as well. For
now "proxy_cache_use_stale updating" is available.


Modified:
trunk/src/http/modules/ngx_http_proxy_module.c
trunk/src/http/ngx_http_cache.h
trunk/src/http/ngx_http_file_cache.c
trunk/src/http/ngx_http_upstream.c
trunk/src/http/ngx_http_upstream.h

Modified: trunk/src/http/modules/ngx_http_proxy_module.c
===================================================================
--- trunk/src/http/modules/ngx_http_proxy_module.c 2011-12-26 10:51:24 UTC (rev 4385)
+++ trunk/src/http/modules/ngx_http_proxy_module.c 2011-12-26 11:15:23 UTC (rev 4386)
@@ -402,6 +402,20 @@
offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_methods),
&ngx_http_upstream_cache_method_mask },

+ { ngx_string("proxy_cache_lock"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_lock),
+ NULL },
+
+ { ngx_string("proxy_cache_lock_timeout"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_msec_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_lock_timeout),
+ NULL },
+
#endif

{ ngx_string("proxy_temp_path"),
@@ -2435,6 +2449,8 @@
conf->upstream.cache_bypass = NGX_CONF_UNSET_PTR;
conf->upstream.no_cache = NGX_CONF_UNSET_PTR;
conf->upstream.cache_valid = NGX_CONF_UNSET_PTR;
+ conf->upstream.cache_lock = NGX_CONF_UNSET;
+ conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC;
#endif

conf->upstream.hide_headers = NGX_CONF_UNSET_PTR;
@@ -2681,6 +2697,12 @@
conf->cache_key = prev->cache_key;
}

+ ngx_conf_merge_value(conf->upstream.cache_lock,
+ prev->upstream.cache_lock, 0);
+
+ ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout,
+ prev->upstream.cache_lock_timeout, 5000);
+
#endif

if (conf->method.len == 0) {

Modified: trunk/src/http/ngx_http_cache.h
===================================================================
--- trunk/src/http/ngx_http_cache.h 2011-12-26 10:51:24 UTC (rev 4385)
+++ trunk/src/http/ngx_http_cache.h 2011-12-26 11:15:23 UTC (rev 4386)
@@ -79,6 +79,14 @@
ngx_http_file_cache_t *file_cache;
ngx_http_file_cache_node_t *node;

+ ngx_msec_t lock_timeout;
+ ngx_msec_t wait_time;
+
+ ngx_event_t wait_event;
+
+ unsigned lock:1;
+ unsigned waiting:1;
+
unsigned updated:1;
unsigned updating:1;
unsigned exists:1;

Modified: trunk/src/http/ngx_http_file_cache.c
===================================================================
--- trunk/src/http/ngx_http_file_cache.c 2011-12-26 10:51:24 UTC (rev 4385)
+++ trunk/src/http/ngx_http_file_cache.c 2011-12-26 11:15:23 UTC (rev 4386)
@@ -10,6 +10,9 @@
#include <ngx_md5.h>


+static ngx_int_t ngx_http_file_cache_lock(ngx_http_request_t *r,
+ ngx_http_cache_t *c);
+static void ngx_http_file_cache_lock_wait_handler(ngx_event_t *ev);
static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r,
ngx_http_cache_t *c);
static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r,
@@ -181,13 +184,13 @@
return NGX_ERROR;
}

+ cln->handler = ngx_http_file_cache_cleanup;
+ cln->data = c;
+
if (ngx_http_file_cache_exists(cache, c) == NGX_ERROR) {
return NGX_ERROR;
}

- cln->handler = ngx_http_file_cache_cleanup;
- cln->data = c;
-
if (ngx_http_file_cache_name(r, cache->path) != NGX_OK) {
return NGX_ERROR;
}
@@ -244,15 +247,24 @@

c = r->cache;

+ if (c->waiting) {
+ return NGX_AGAIN;
+ }
+
if (c->buf) {
return ngx_http_file_cache_read(r, c);
}

cache = c->file_cache;

- cln = ngx_pool_cleanup_add(r->pool, 0);
- if (cln == NULL) {
- return NGX_ERROR;
+ if (c->node == NULL) {
+ cln = ngx_pool_cleanup_add(r->pool, 0);
+ if (cln == NULL) {
+ return NGX_ERROR;
+ }
+
+ cln->handler = ngx_http_file_cache_cleanup;
+ cln->data = c;
}

rc = ngx_http_file_cache_exists(cache, c);
@@ -264,9 +276,6 @@
return rc;
}

- cln->handler = ngx_http_file_cache_cleanup;
- cln->data = c;
-
if (rc == NGX_AGAIN) {
return NGX_HTTP_CACHE_SCARCE;
}
@@ -306,7 +315,7 @@
}

if (!test) {
- return NGX_DECLINED;
+ goto done;
}

clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
@@ -330,7 +339,7 @@

case NGX_ENOENT:
case NGX_ENOTDIR:
- return rv;
+ goto done;

default:
ngx_log_error(NGX_LOG_CRIT, r->connection->log, of.err,
@@ -354,10 +363,118 @@
}

return ngx_http_file_cache_read(r, c);
+
+done:
+
+ if (rv == NGX_DECLINED) {
+ return ngx_http_file_cache_lock(r, c);
+ }
+
+ return rv;
}


static ngx_int_t
+ngx_http_file_cache_lock(ngx_http_request_t *r, ngx_http_cache_t *c)
+{
+ ngx_msec_t now, timer;
+ ngx_http_file_cache_t *cache;
+
+ if (!c->lock) {
+ return NGX_DECLINED;
+ }
+
+ cache = c->file_cache;
+
+ ngx_shmtx_lock(&cache->shpool->mutex);
+
+ if (!c->node->updating) {
+ c->node->updating = 1;
+ c->updating = 1;
+ }
+
+ ngx_shmtx_unlock(&cache->shpool->mutex);
+
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "http file cache lock u:%d wt:%M",
+ c->updating, c->wait_time);
+
+ if (c->updating) {
+ return NGX_DECLINED;
+ }
+
+ c->waiting = 1;
+
+ now = ngx_current_msec;
+
+ if (c->wait_time == 0) {
+ c->wait_time = now + c->lock_timeout;
+
+ c->wait_event.handler = ngx_http_file_cache_lock_wait_handler;
+ c->wait_event.data = r;
+ c->wait_event.log = r->connection->log;
+ }
+
+ timer = c->wait_time - now;
+
+ ngx_add_timer(&c->wait_event, (timer > 500) ? 500 : timer);
+
+ r->main->blocked++;
+
+ return NGX_AGAIN;
+}
+
+
+static void
+ngx_http_file_cache_lock_wait_handler(ngx_event_t *ev)
+{
+ ngx_uint_t wait;
+ ngx_msec_t timer;
+ ngx_http_cache_t *c;
+ ngx_http_request_t *r;
+ ngx_http_file_cache_t *cache;
+
+ r = ev->data;
+ c = r->cache;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0,
+ "http file cache wait handler wt:%M cur:%M",
+ c->wait_time, ngx_current_msec);
+
+ timer = c->wait_time - ngx_current_msec;
+
+ if ((ngx_msec_int_t) timer <= 0) {
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0,
+ "http file cache lock timeout");
+ c->lock = 0;
+ goto wakeup;
+ }
+
+ cache = c->file_cache;
+ wait = 0;
+
+ ngx_shmtx_lock(&cache->shpool->mutex);
+
+ if (c->node->updating) {
+ wait = 1;
+ }
+
+ ngx_shmtx_unlock(&cache->shpool->mutex);
+
+ if (wait) {
+ ngx_add_timer(ev, (timer > 500) ? 500 : timer);
+ return;
+ }
+
+wakeup:
+
+ c->waiting = 0;
+ r->main->blocked--;
+ r->connection->write->handler(r->connection->write);
+}
+
+
+static ngx_int_t
ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c)
{
time_t now;
@@ -518,13 +635,19 @@

ngx_shmtx_lock(&cache->shpool->mutex);

- fcn = ngx_http_file_cache_lookup(cache, c->key);
+ fcn = c->node;

+ if (fcn == NULL) {
+ fcn = ngx_http_file_cache_lookup(cache, c->key);
+ }
+
if (fcn) {
ngx_queue_remove(&fcn->queue);

- fcn->uses++;
- fcn->count++;
+ if (c->node == NULL) {
+ fcn->uses++;
+ fcn->count++;
+ }

if (fcn->error) {

@@ -621,6 +744,10 @@

c = r->cache;

+ if (c->file.name.len) {
+ return NGX_OK;
+ }
+
c->file.name.len = path->name.len + 1 + path->len
+ 2 * NGX_HTTP_CACHE_KEY_LEN;

@@ -957,6 +1084,10 @@
}
}
}
+
+ if (c->wait_event.timer_set) {
+ ngx_del_timer(&c->wait_event);
+ }
}



Modified: trunk/src/http/ngx_http_upstream.c
===================================================================
--- trunk/src/http/ngx_http_upstream.c 2011-12-26 10:51:24 UTC (rev 4385)
+++ trunk/src/http/ngx_http_upstream.c 2011-12-26 11:15:23 UTC (rev 4386)
@@ -707,6 +707,9 @@
c->body_start = u->conf->buffer_size;
c->file_cache = u->conf->cache->data;

+ c->lock = u->conf->cache_lock;
+ c->lock_timeout = u->conf->cache_lock_timeout;
+
u->cache_status = NGX_HTTP_CACHE_MISS;
}


Modified: trunk/src/http/ngx_http_upstream.h
===================================================================
--- trunk/src/http/ngx_http_upstream.h 2011-12-26 10:51:24 UTC (rev 4385)
+++ trunk/src/http/ngx_http_upstream.h 2011-12-26 11:15:23 UTC (rev 4386)
@@ -165,6 +165,9 @@
ngx_uint_t cache_use_stale;
ngx_uint_t cache_methods;

+ ngx_flag_t cache_lock;
+ ngx_msec_t cache_lock_timeout;
+
ngx_array_t *cache_valid;
ngx_array_t *cache_bypass;
ngx_array_t *no_cache;

_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel