Welcome! Log In Create A New Profile

Advanced

[PATCH 6 of 6] Upstream: reschedule in-progress resolve tasks at worker exit

Aleksei Bavshin
January 31, 2023 08:42PM
# HG changeset patch
# User Aleksei Bavshin <a.bavshin@nginx.com>
# Date 1671568350 28800
# Tue Dec 20 12:32:30 2022 -0800
# Node ID 34e439843ed3a2122cba54b1f40b77ea6e874078
# Parent f8eb6b94d8f46008eb5f2f1dbc747750d5755506
Upstream: reschedule in-progress resolve tasks at worker exit.

Workers may exit at a different time, depending on the active
connections and other factors. If the peer was being resolved in one
of the workers at the moment of termination, it won't be returned to
the resolve queue and remaining workers won't be able to continue
updating it.

To address that we can disown all interrupted resolve tasks and put
them back to the queue at worker exit. The same logic runs on the
process init to recover tasks affected by a worker crash.

diff --git a/src/http/modules/ngx_http_upstream_zone_module.c b/src/http/modules/ngx_http_upstream_zone_module.c
--- a/src/http/modules/ngx_http_upstream_zone_module.c
+++ b/src/http/modules/ngx_http_upstream_zone_module.c
@@ -40,7 +40,10 @@ static ngx_command_t ngx_http_upstream_
};


+static ngx_uint_t ngx_http_upstream_zone_recover_peers(
+ ngx_http_upstream_srv_conf_t *uscf);
static ngx_int_t ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle);
+static void ngx_http_upstream_zone_exit_worker(ngx_cycle_t *cycle);
static void ngx_http_upstream_zone_resolve_timer(ngx_event_t *event);
static void ngx_http_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx);
static void ngx_http_upstream_zone_resolve_queue_insert(ngx_queue_t *queue,
@@ -77,7 +80,7 @@ ngx_module_t ngx_http_upstream_zone_mod
ngx_http_upstream_zone_init_worker, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
- NULL, /* exit process */
+ ngx_http_upstream_zone_exit_worker, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING
};
@@ -573,10 +576,51 @@ ngx_http_upstream_zone_resolve_timer(ngx
}


+static ngx_uint_t
+ngx_http_upstream_zone_recover_peers(ngx_http_upstream_srv_conf_t *uscf)
+{
+ ngx_msec_t now;
+ ngx_uint_t n;
+ ngx_http_upstream_host_t *host;
+ ngx_http_upstream_rr_peer_t *peer;
+ ngx_http_upstream_rr_peers_t *peers;
+
+ n = 0;
+ now = ngx_current_msec;
+ peers = uscf->peer.data;
+
+ do {
+ ngx_http_upstream_rr_peers_wlock(peers);
+
+ for (peer = peers->resolve; peer; peer = peer->next) {
+
+ host = peer->host;
+
+ if (host->worker != ngx_worker) {
+ continue;
+ }
+
+ host->expires = now;
+ host->worker = NGX_UPSTREAM_RESOLVE_NO_WORKER;
+ ngx_queue_insert_head(&peers->resolve_queue, &host->queue);
+
+ n++;
+ }
+
+ ngx_http_upstream_rr_peers_unlock(peers);
+
+ peers = peers->next;
+
+ } while (peers);
+
+ return n;
+}
+
+
static ngx_int_t
ngx_http_upstream_zone_init_worker(ngx_cycle_t *cycle)
{
- ngx_uint_t i;
+ ngx_uint_t i, n;
ngx_event_t *event;
ngx_http_upstream_srv_conf_t *uscf, **uscfp;
ngx_http_upstream_main_conf_t *umcf;
@@ -594,6 +638,7 @@ ngx_http_upstream_zone_init_worker(ngx_c
}

uscfp = umcf->upstreams.elts;
+ n = 0;

for (i = 0; i < umcf->upstreams.nelts; i++) {

@@ -603,6 +648,8 @@ ngx_http_upstream_zone_init_worker(ngx_c
continue;
}

+ n += ngx_http_upstream_zone_recover_peers(uscf);
+
event = &uscf->event;
event->data = uscf;
event->handler = ngx_http_upstream_zone_resolve_timer;
@@ -612,11 +659,56 @@ ngx_http_upstream_zone_init_worker(ngx_c
ngx_add_timer(event, 1);
}

+ if (n) {
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, cycle->log, 0,
+ "upstream: recovered %ui resolvable peers", n);
+ }
+
return NGX_OK;
}


static void
+ngx_http_upstream_zone_exit_worker(ngx_cycle_t *cycle)
+{
+ ngx_uint_t i, n;
+ ngx_http_upstream_srv_conf_t *uscf, **uscfp;
+ ngx_http_upstream_main_conf_t *umcf;
+
+ if (ngx_process != NGX_PROCESS_WORKER
+ && ngx_process != NGX_PROCESS_SINGLE)
+ {
+ return;
+ }
+
+ umcf = ngx_http_cycle_get_module_main_conf(cycle, ngx_http_upstream_module);
+
+ if (umcf == NULL) {
+ return;
+ }
+
+ uscfp = umcf->upstreams.elts;
+ n = 0;
+
+ for (i = 0; i < umcf->upstreams.nelts; i++) {
+
+ uscf = uscfp[i];
+
+ if (uscf->shm_zone == NULL) {
+ continue;
+ }
+
+ n += ngx_http_upstream_zone_recover_peers(uscf);
+ }
+
+ if (n) {
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, cycle->log, 0,
+ "upstream: released %ui resolvable peers", n);
+ }
+}
+
+
+static void
ngx_http_upstream_zone_start_resolve(ngx_http_upstream_srv_conf_t *uscf,
ngx_http_upstream_host_t *host)
{
diff --git a/src/stream/ngx_stream_upstream_zone_module.c b/src/stream/ngx_stream_upstream_zone_module.c
--- a/src/stream/ngx_stream_upstream_zone_module.c
+++ b/src/stream/ngx_stream_upstream_zone_module.c
@@ -40,7 +40,10 @@ static ngx_command_t ngx_stream_upstrea
};


+static ngx_uint_t ngx_stream_upstream_zone_recover_peers(
+ ngx_stream_upstream_srv_conf_t *uscf);
static ngx_int_t ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle);
+static void ngx_stream_upstream_zone_exit_worker(ngx_cycle_t *cycle);
static void ngx_stream_upstream_zone_resolve_timer(ngx_event_t *event);
static void ngx_stream_upstream_zone_resolve_handler(ngx_resolver_ctx_t *ctx);
static void ngx_stream_upstream_zone_resolve_queue_insert(ngx_queue_t *queue,
@@ -74,7 +77,7 @@ ngx_module_t ngx_stream_upstream_zone_m
ngx_stream_upstream_zone_init_worker, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
- NULL, /* exit process */
+ ngx_stream_upstream_zone_exit_worker, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING
};
@@ -570,10 +573,51 @@ ngx_stream_upstream_zone_resolve_timer(n
}


+static ngx_uint_t
+ngx_stream_upstream_zone_recover_peers(ngx_stream_upstream_srv_conf_t *uscf)
+{
+ ngx_msec_t now;
+ ngx_uint_t n;
+ ngx_stream_upstream_host_t *host;
+ ngx_stream_upstream_rr_peer_t *peer;
+ ngx_stream_upstream_rr_peers_t *peers;
+
+ n = 0;
+ now = ngx_current_msec;
+ peers = uscf->peer.data;
+
+ do {
+ ngx_stream_upstream_rr_peers_wlock(peers);
+
+ for (peer = peers->resolve; peer; peer = peer->next) {
+
+ host = peer->host;
+
+ if (host->worker != ngx_worker) {
+ continue;
+ }
+
+ host->expires = now;
+ host->worker = NGX_UPSTREAM_RESOLVE_NO_WORKER;
+ ngx_queue_insert_head(&peers->resolve_queue, &host->queue);
+
+ n++;
+ }
+
+ ngx_stream_upstream_rr_peers_unlock(peers);
+
+ peers = peers->next;
+
+ } while (peers);
+
+ return n;
+}
+
+
static ngx_int_t
ngx_stream_upstream_zone_init_worker(ngx_cycle_t *cycle)
{
- ngx_uint_t i;
+ ngx_uint_t i, n;
ngx_event_t *event;
ngx_stream_upstream_srv_conf_t *uscf, **uscfp;
ngx_stream_upstream_main_conf_t *umcf;
@@ -592,6 +636,7 @@ ngx_stream_upstream_zone_init_worker(ngx
}

uscfp = umcf->upstreams.elts;
+ n = 0;

for (i = 0; i < umcf->upstreams.nelts; i++) {

@@ -601,6 +646,8 @@ ngx_stream_upstream_zone_init_worker(ngx
continue;
}

+ n += ngx_stream_upstream_zone_recover_peers(uscf);
+
event = &uscf->event;
event->data = uscf;
event->handler = ngx_stream_upstream_zone_resolve_timer;
@@ -610,11 +657,57 @@ ngx_stream_upstream_zone_init_worker(ngx
ngx_add_timer(event, 1);
}

+ if (n) {
+ ngx_log_debug1(NGX_LOG_DEBUG_STREAM, cycle->log, 0,
+ "upstream: recovered %ui resolvable peers", n);
+ }
+
return NGX_OK;
}


static void
+ngx_stream_upstream_zone_exit_worker(ngx_cycle_t *cycle)
+{
+ ngx_uint_t i, n;
+ ngx_stream_upstream_srv_conf_t *uscf, **uscfp;
+ ngx_stream_upstream_main_conf_t *umcf;
+
+ if (ngx_process != NGX_PROCESS_WORKER
+ && ngx_process != NGX_PROCESS_SINGLE)
+ {
+ return;
+ }
+
+ umcf = ngx_stream_cycle_get_module_main_conf(cycle,
+ ngx_stream_upstream_module);
+
+ if (umcf == NULL) {
+ return;
+ }
+
+ uscfp = umcf->upstreams.elts;
+ n = 0;
+
+ for (i = 0; i < umcf->upstreams.nelts; i++) {
+
+ uscf = uscfp[i];
+
+ if (uscf->shm_zone == NULL) {
+ continue;
+ }
+
+ n += ngx_stream_upstream_zone_recover_peers(uscf);
+ }
+
+ if (n) {
+ ngx_log_debug1(NGX_LOG_DEBUG_STREAM, cycle->log, 0,
+ "upstream: released %ui resolvable peers", n);
+ }
+}
+
+
+static void
ngx_stream_upstream_zone_start_resolve(ngx_stream_upstream_srv_conf_t *uscf,
ngx_stream_upstream_host_t *host)
{
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel
Subject Author Views Posted

[PATCH 0 of 6] Upstream: re-resolvable servers.

Aleksei Bavshin 559 January 31, 2023 08:40PM

[PATCH 1 of 6] Upstream: re-resolvable servers

Aleksei Bavshin 192 January 31, 2023 08:40PM

[PATCH 2 of 6] Stream: re-resolvable servers

Aleksei Bavshin 151 January 31, 2023 08:40PM

[PATCH 3 of 6] Upstream: construct upstream peers from DNS SRV records

Aleksei Bavshin 170 January 31, 2023 08:40PM

[PATCH 4 of 6] Upstream: per-upstream resolver

Aleksei Bavshin 158 January 31, 2023 08:42PM

[PATCH 5 of 6] Upstream: allow any worker to resolve upstream servers

Aleksei Bavshin 146 January 31, 2023 08:42PM

Re: [PATCH 5 of 6] Upstream: allow any worker to resolve upstream servers

J Carter 188 February 05, 2023 10:02PM

Re: [PATCH 5 of 6] Upstream: allow any worker to resolve upstream servers

Aleksei Bavshin 131 February 09, 2023 11:46AM

Re: [PATCH 5 of 6] Upstream: allow any worker to resolve upstream servers

J Carter 158 February 09, 2023 07:06PM

[PATCH 6 of 6] Upstream: reschedule in-progress resolve tasks at worker exit

Aleksei Bavshin 136 January 31, 2023 08:42PM



Sorry, you do not have permission to post/reply in this forum.

Online Users

Guests: 298
Record Number of Users: 8 on April 13, 2023
Record Number of Guests: 421 on December 02, 2018
Powered by nginx      Powered by FreeBSD      PHP Powered      Powered by MariaDB      ipv6 ready