Welcome! Log In Create A New Profile

Advanced

cache: move open to thread pool

Ka-Hing Cheung via nginx-devel
August 07, 2018 05:28PM
commit 0bcfefa040abdff674047c15701d237ff16a5f2b
Author: Ka-Hing Cheung <kahing@cloudflare.com>
Date: Fri Aug 3 13:37:58 2018 -0700

move open to thread pool

At cloudflare we found that open() can block a long time, especially
at p99 and p999. Moving it to thread pool improved overall p99 by 6x
or so during peak time.

This has the side effect of disabling nginx's open_file_cache when
"aio threads" is turned on. For us we found that to have no impact
(probably because we have way too many files for open file cache to
be effective anyway).

thread pool does increase CPU usage somewhat but we have not measured
difference in CPU usage for stock "aio threads" compared to after this
patch.

Only the cache hit open() is moved to thread pool.

We wrote more about it here:
https://blog.cloudflare.com/how-we-scaled-nginx-and-saved-the-world-54-years-every-day/

diff --git src/core/ngx_open_file_cache.c src/core/ngx_open_file_cache.c
index b23ee78d..9177ebfd 100644
--- src/core/ngx_open_file_cache.c
+++ src/core/ngx_open_file_cache.c
@@ -35,8 +35,6 @@ static ngx_fd_t ngx_open_file_wrapper(ngx_str_t *name,
ngx_int_t access, ngx_log_t *log);
static ngx_int_t ngx_file_info_wrapper(ngx_str_t *name,
ngx_open_file_info_t *of, ngx_file_info_t *fi, ngx_log_t *log);
-static ngx_int_t ngx_open_and_stat_file(ngx_str_t *name,
- ngx_open_file_info_t *of, ngx_log_t *log);
static void ngx_open_file_add_event(ngx_open_file_cache_t *cache,
ngx_cached_open_file_t *file, ngx_open_file_info_t *of, ngx_log_t *log);
static void ngx_open_file_cleanup(void *data);
@@ -836,7 +834,7 @@ ngx_file_info_wrapper(ngx_str_t *name,
ngx_open_file_info_t *of,
}


-static ngx_int_t
+ngx_int_t
ngx_open_and_stat_file(ngx_str_t *name, ngx_open_file_info_t *of,
ngx_log_t *log)
{
diff --git src/core/ngx_open_file_cache.h src/core/ngx_open_file_cache.h
index d119c129..94b4d552 100644
--- src/core/ngx_open_file_cache.h
+++ src/core/ngx_open_file_cache.h
@@ -124,6 +124,8 @@ ngx_open_file_cache_t
*ngx_open_file_cache_init(ngx_pool_t *pool,
ngx_uint_t max, time_t inactive);
ngx_int_t ngx_open_cached_file(ngx_open_file_cache_t *cache, ngx_str_t *name,
ngx_open_file_info_t *of, ngx_pool_t *pool);
+ngx_int_t ngx_open_and_stat_file(ngx_str_t *name,
+ ngx_open_file_info_t *of, ngx_log_t *log);


#endif /* _NGX_OPEN_FILE_CACHE_H_INCLUDED_ */
diff --git src/http/ngx_http_cache.h src/http/ngx_http_cache.h
index f9e96640..2910f6a1 100644
--- src/http/ngx_http_cache.h
+++ src/http/ngx_http_cache.h
@@ -114,6 +114,7 @@ struct ngx_http_cache_s {
unsigned exists:1;
unsigned temp_file:1;
unsigned purged:1;
+ unsigned opening:1;
unsigned reading:1;
unsigned secondary:1;
unsigned background:1;
diff --git src/http/ngx_http_file_cache.c src/http/ngx_http_file_cache.c
index 56866fa4..9cb110bf 100644
--- src/http/ngx_http_file_cache.c
+++ src/http/ngx_http_file_cache.c
@@ -18,6 +18,10 @@ static void
ngx_http_file_cache_lock_wait(ngx_http_request_t *r,
ngx_http_cache_t *c);
static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r,
ngx_http_cache_t *c);
+static ngx_int_t ngx_http_file_cache_aio_open(ngx_http_request_t *r,
+ ngx_http_cache_t *c, ngx_int_t *rv);
+static ngx_int_t ngx_http_file_cache_do_aio_open(ngx_http_request_t *r,
+ ngx_http_cache_t *c, ngx_open_file_info_t *of, ngx_int_t *rv);
static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r,
ngx_http_cache_t *c);
#if (NGX_HAVE_FILE_AIO)
@@ -268,9 +272,7 @@ ngx_http_file_cache_open(ngx_http_request_t *r)
ngx_uint_t test;
ngx_http_cache_t *c;
ngx_pool_cleanup_t *cln;
- ngx_open_file_info_t of;
ngx_http_file_cache_t *cache;
- ngx_http_core_loc_conf_t *clcf;

c = r->cache;

@@ -278,6 +280,10 @@ ngx_http_file_cache_open(ngx_http_request_t *r)
return NGX_AGAIN;
}

+ if (c->opening) {
+ return ngx_http_file_cache_aio_open(r, c, &rv);
+ }
+
if (c->reading) {
return ngx_http_file_cache_read(r, c);
}
@@ -343,23 +349,33 @@ ngx_http_file_cache_open(ngx_http_request_t *r)
goto done;
}

- clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+ return ngx_http_file_cache_aio_open(r, c, &rv);

- ngx_memzero(&of, sizeof(ngx_open_file_info_t));
+done:

- of.uniq = c->uniq;
- of.valid = clcf->open_file_cache_valid;
- of.min_uses = clcf->open_file_cache_min_uses;
- of.events = clcf->open_file_cache_events;
- of.directio = NGX_OPEN_FILE_DIRECTIO_OFF;
- of.read_ahead = clcf->read_ahead;
+ if (rv == NGX_DECLINED) {
+ return ngx_http_file_cache_lock(r, c);
+ }

- if (ngx_open_cached_file(clcf->open_file_cache, &c->file.name,
&of, r->pool)
- != NGX_OK)
- {
- switch (of.err) {
+ return rv;
+}
+
+static ngx_int_t
+ngx_http_file_cache_aio_open(ngx_http_request_t *r, ngx_http_cache_t *c,
+ ngx_int_t *rv)
+{
+ ngx_int_t rc;
+ ngx_open_file_info_t of;
+ ngx_http_file_cache_t *cache = c->file_cache;

- case 0:
+ rc = ngx_http_file_cache_do_aio_open(r, c, &of, rv);
+ if (rc == NGX_AGAIN) {
+ return rc;
+ }
+
+ if (rc != NGX_OK) {
+ switch (of.err) {
+ case NGX_OK:
return NGX_ERROR;

case NGX_ENOENT:
@@ -391,14 +407,13 @@ ngx_http_file_cache_open(ngx_http_request_t *r)

done:

- if (rv == NGX_DECLINED) {
+ if (*rv == NGX_DECLINED) {
return ngx_http_file_cache_lock(r, c);
}

- return rv;
+ return *rv;
}

-
static ngx_int_t
ngx_http_file_cache_lock(ngx_http_request_t *r, ngx_http_cache_t *c)
{
@@ -663,6 +678,127 @@ ngx_http_file_cache_read(ngx_http_request_t *r,
ngx_http_cache_t *c)
}


+#if (NGX_THREADS)
+typedef struct {
+ ngx_str_t name;
+ ngx_pool_t *pool;
+ ngx_open_file_info_t of;
+ ngx_int_t rv;
+} ngx_thread_file_open_ctx_t;
+
+static void
+ngx_http_file_cache_thread_open_handler(void *data, ngx_log_t *log)
+{
+ ngx_thread_file_open_ctx_t *ctx = data;
+ ngx_pool_t *pool = ctx->pool;
+ ngx_open_file_info_t *of = &ctx->of;
+ ngx_pool_cleanup_t *cln;
+ ngx_pool_cleanup_file_t *clnf;
+ ngx_int_t rc;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "thread read handler");
+
+ cln = ngx_pool_cleanup_add(pool, sizeof(ngx_pool_cleanup_file_t));
+ if (cln == NULL) {
+ ctx->of.err = NGX_ERROR;
+ return;
+ }
+
+ of->fd = NGX_INVALID_FILE;
+
+ rc = ngx_open_and_stat_file(&ctx->name, of, pool->log);
+ if (rc == NGX_OK && !of->is_dir) {
+ cln->handler = ngx_pool_cleanup_file;
+ clnf = cln->data;
+
+ clnf->fd = of->fd;
+ clnf->name = ctx->name.data;
+ clnf->log = pool->log;
+ }
+}
+
+
+static ngx_int_t
+ngx_http_file_cache_thread_open(ngx_http_cache_t *c, ngx_open_file_info_t *of,
+ ngx_int_t *rv, ngx_pool_t *pool)
+{
+ ngx_thread_task_t *task;
+ ngx_thread_file_open_ctx_t *ctx;
+ ngx_file_t *file = &c->file;
+
+ task = file->thread_task;
+
+ if (task == NULL) {
+ task = ngx_thread_task_alloc(pool, sizeof(ngx_thread_file_open_ctx_t));
+ if (task == NULL) {
+ return NGX_ERROR;
+ }
+
+ file->thread_task = task;
+ }
+
+ ctx = task->ctx;
+
+ if (task->event.complete) {
+ task->event.complete = 0;
+
+ *of = ctx->of;
+ *rv = ctx->rv;
+ return of->err;
+ }
+
+ task->handler = ngx_http_file_cache_thread_open_handler;
+
+ ctx->pool = pool;
+ ctx->name = file->name;
+ ctx->of = *of;
+ ctx->rv = *rv;
+
+ if (file->thread_handler(task, file) != NGX_OK) {
+ return NGX_ERROR;
+ }
+
+ return NGX_AGAIN;
+}
+#endif
+
+static ngx_int_t
+ngx_http_file_cache_do_aio_open(ngx_http_request_t *r, ngx_http_cache_t *c,
+ ngx_open_file_info_t *of, ngx_int_t *rv)
+{
+ ngx_http_core_loc_conf_t *clcf;
+
+ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+
+ ngx_memzero(of, sizeof(ngx_open_file_info_t));
+
+ of->uniq = c->uniq;
+ of->valid = clcf->open_file_cache_valid;
+ of->min_uses = clcf->open_file_cache_min_uses;
+ of->events = clcf->open_file_cache_events;
+ of->directio = NGX_OPEN_FILE_DIRECTIO_OFF;
+ of->read_ahead = clcf->read_ahead;
+
+#if (NGX_THREADS)
+ if (clcf->aio == NGX_HTTP_AIO_THREADS) {
+ ngx_int_t rc;
+
+ c->file.thread_task = c->thread_task;
+ c->file.thread_handler = ngx_http_cache_thread_handler;
+ c->file.thread_ctx = r;
+
+ rc = ngx_http_file_cache_thread_open(c, of, rv, r->pool);
+
+ c->thread_task = c->file.thread_task;
+ c->opening = (rc == NGX_AGAIN);
+
+ return rc;
+ }
+#endif
+
+ return ngx_open_cached_file(clcf->open_file_cache, &c->file.name,
of, r->pool);
+}
+
static ssize_t
ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c)
{
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel
Subject Author Views Posted

cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 2961 August 07, 2018 05:28PM

Re: cache: move open to thread pool

Maxim Dounin 928 August 08, 2018 02:18PM

RE: cache: move open to thread pool

erankor 950 August 08, 2018 03:46PM

Re: cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 700 August 27, 2018 06:00PM

RE: cache: move open to thread pool

erankor 912 August 28, 2018 05:18AM

Re: cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 715 August 09, 2018 05:44PM

Re: cache: move open to thread pool

Maxim Dounin 1175 August 10, 2018 07:42AM

Re: cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 739 August 27, 2018 05:40PM

Re: cache: move open to thread pool

Maxim Dounin 828 September 03, 2018 12:10PM

Re: cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 766 September 04, 2018 08:00PM

Re: cache: move open to thread pool

Maxim Dounin 726 September 13, 2018 02:12PM

Re: cache: move open to thread pool

Roman Arutyunyan 844 October 04, 2018 05:34AM

Re: cache: move open to thread pool

Roman Arutyunyan 832 November 01, 2018 08:30AM

Re: cache: move open to thread pool

Maxim Konovalov 692 November 01, 2018 12:08PM

Re: cache: move open to thread pool

Roman Arutyunyan 556 August 03, 2021 04:52AM

Re: cache: move open to thread pool

Maxim Konovalov 691 November 08, 2018 09:20AM

Re: cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 730 November 15, 2018 05:54PM

Re: cache: move open to thread pool

Maxim Konovalov 779 November 16, 2018 04:12AM

Re: cache: move open to thread pool

Maxim Konovalov 755 December 05, 2018 04:44AM

Re: cache: move open to thread pool

Maxim Konovalov 700 January 10, 2019 03:32AM

Re: cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 747 January 14, 2019 08:54PM

Re: cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 655 January 22, 2019 01:22PM

Re: cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 725 January 22, 2019 02:36PM

Re: cache: move open to thread pool

Maxim Konovalov 704 January 23, 2019 07:40AM

Re: cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 680 February 06, 2019 04:20PM

Re: cache: move open to thread pool

Maxim Konovalov 717 February 07, 2019 08:40AM

Re: cache: move open to thread pool

Ka-Hing Cheung via nginx-devel 767 February 08, 2019 05:42PM

Re: cache: move open to thread pool

karton 606 July 19, 2019 11:28AM

Re: cache: move open to thread pool

Maxim Konovalov 642 July 19, 2019 02:44PM

Re: cache: move open to thread pool

karton 566 July 19, 2019 09:46PM

Re: cache: move open to thread pool

Roman Arutyunyan 630 July 22, 2019 10:06AM

Re: cache: move open to thread pool

karton 612 July 22, 2019 11:40AM

Re: cache: move open to thread pool

karton 617 July 23, 2019 11:48PM

Re: cache: move open to thread pool

Roman Arutyunyan 677 August 16, 2019 10:26AM

Re: cache: move open to thread pool

karton 747 August 16, 2019 10:52AM

Re: cache: move open to thread pool

NCviQwilt 591 September 02, 2021 12:22PM

Re: cache: move open to thread pool

Roman Arutyunyan 627 September 09, 2021 06:50AM



Sorry, you do not have permission to post/reply in this forum.

Online Users

Guests: 200
Record Number of Users: 8 on April 13, 2023
Record Number of Guests: 500 on July 15, 2024
Powered by nginx      Powered by FreeBSD      PHP Powered      Powered by MariaDB      ipv6 ready