details: https://github.com/nginx/nginx/commit/5ebe7a4122c9653ed6b06e6577fc68904ad061c4
branches: master
commit: 5ebe7a4122c9653ed6b06e6577fc68904ad061c4
user: Ruslan Ermilov <ru@nginx.com>
date: Fri, 3 Nov 2017 22:22:23 +0300
description:
Upstream: pre-resolve servers on reload.

After configuration is reloaded, it may take some time for the
re-resolvable upstream servers to resolve and become available
as peers. During this time, client requests might get dropped.

Such servers are now pre-resolved using the "cache" of already
resolved peers from the old shared memory zone.
---
src/http/modules/ngx_http_upstream_zone_module.c | 201 ++++++++++++++++++++++-
src/stream/ngx_stream_upstream_zone_module.c | 201 ++++++++++++++++++++++-
2 files changed, 388 insertions(+), 14 deletions(-)

diff --git a/src/http/modules/ngx_http_upstream_zone_module.c b/src/http/modules/ngx_http_upstream_zone_module.c
index e3ede9268..c1931a8bb 100644
--- a/src/http/modules/ngx_http_upstream_zone_module.c
+++ b/src/http/modules/ngx_http_upstream_zone_module.c
@@ -15,9 +15,15 @@ static char *ngx_http_upstream_zone(ngx_conf_t *cf, ngx_command_t *cmd,
static ngx_int_t ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone,
void *data);
static ngx_http_upstream_rr_peers_t *ngx_http_upstream_zone_copy_peers(
- ngx_slab_pool_t *shpool, ngx_http_upstream_srv_conf_t *uscf);
+ ngx_slab_pool_t *shpool, ngx_http_upstream_srv_conf_t *uscf,
+ ngx_http_upstream_srv_conf_t *ouscf);
static ngx_http_upstream_rr_peer_t *ngx_http_upstream_zone_copy_peer(
ngx_http_upstream_rr_peers_t *peers, ngx_http_upstream_rr_peer_t *src);
+static ngx_int_t ngx_http_upstream_zone_preresolve(
+ ngx_http_upstream_rr_peer_t *resolve,
+ ngx_http_upstream_rr_peers_t *peers,
+ ngx_http_upstream_rr_peer_t *oresolve,
+ ngx_http_upstream_rr_peers_t *opeers);
static void ngx_http_upstream_zone_set_single(
ngx_http_upstream_srv_conf_t *uscf);
static void ngx_http_upstream_zone_remove_peer_locked(
@@ -128,11 +134,11 @@ static ngx_int_t
ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
{
size_t len;
- ngx_uint_t i;
+ ngx_uint_t i, j;
ngx_slab_pool_t *shpool;
ngx_http_upstream_rr_peers_t *peers, **peersp;
- ngx_http_upstream_srv_conf_t *uscf, **uscfp;
- ngx_http_upstream_main_conf_t *umcf;
+ ngx_http_upstream_srv_conf_t *uscf, *ouscf, **uscfp, **ouscfp;
+ ngx_http_upstream_main_conf_t *umcf, *oumcf;

shpool = (ngx_slab_pool_t *) shm_zone->shm.addr;
umcf = shm_zone->data;
@@ -169,6 +175,7 @@ ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
/* copy peers to shared memory */

peersp = (ngx_http_upstream_rr_peers_t **) (void *) &shpool->data;
+ oumcf = data;

for (i = 0; i < umcf->upstreams.nelts; i++) {
uscf = uscfp[i];
@@ -177,7 +184,38 @@ ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
continue;
}

- peers = ngx_http_upstream_zone_copy_peers(shpool, uscf);
+ ouscf = NULL;
+
+ if (oumcf) {
+ ouscfp = oumcf->upstreams.elts;
+
+ for (j = 0; j < oumcf->upstreams.nelts; j++) {
+
+ if (ouscfp[j]->shm_zone == NULL) {
+ continue;
+ }
+
+ if (ouscfp[j]->shm_zone->shm.name.len != shm_zone->shm.name.len
+ || ngx_memcmp(ouscfp[j]->shm_zone->shm.name.data,
+ shm_zone->shm.name.data,
+ shm_zone->shm.name.len)
+ != 0)
+ {
+ continue;
+ }
+
+ if (ouscfp[j]->host.len == uscf->host.len
+ && ngx_memcmp(ouscfp[j]->host.data, uscf->host.data,
+ uscf->host.len)
+ == 0)
+ {
+ ouscf = ouscfp[j];
+ break;
+ }
+ }
+ }
+
+ peers = ngx_http_upstream_zone_copy_peers(shpool, uscf, ouscf);
if (peers == NULL) {
return NGX_ERROR;
}
@@ -192,12 +230,14 @@ ngx_http_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)

static ngx_http_upstream_rr_peers_t *
ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
- ngx_http_upstream_srv_conf_t *uscf)
+ ngx_http_upstream_srv_conf_t *uscf, ngx_http_upstream_srv_conf_t *ouscf)
{
ngx_str_t *name;
ngx_uint_t *config;
ngx_http_upstream_rr_peer_t *peer, **peerp;
- ngx_http_upstream_rr_peers_t *peers, *backup;
+ ngx_http_upstream_rr_peers_t *peers, *opeers, *backup;
+
+ opeers = (ouscf ? ouscf->peer.data : NULL);

config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t));
if (config == NULL) {
@@ -250,6 +290,16 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
(*peers->config)++;
}

+ if (opeers) {
+
+ if (ngx_http_upstream_zone_preresolve(peers->resolve, peers,
+ opeers->resolve, opeers)
+ != NGX_OK)
+ {
+ return NULL;
+ }
+ }
+
if (peers->next == NULL) {
goto done;
}
@@ -289,10 +339,30 @@ ngx_http_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,

peers->next = backup;

+ if (opeers && opeers->next) {
+
+ if (ngx_http_upstream_zone_preresolve(peers->resolve, backup,
+ opeers->resolve, opeers->next)
+ != NGX_OK)
+ {
+ return NULL;
+ }
+
+ if (ngx_http_upstream_zone_preresolve(backup->resolve, backup,
+ opeers->next->resolve,
+ opeers->next)
+ != NGX_OK)
+ {
+ return NULL;
+ }
+ }
+
done:

uscf->peer.data = peers;

+ ngx_http_upstream_zone_set_single(uscf);
+
return peers;
}

@@ -404,6 +474,123 @@ failed:
}


+static ngx_int_t
+ngx_http_upstream_zone_preresolve(ngx_http_upstream_rr_peer_t *resolve,
+ ngx_http_upstream_rr_peers_t *peers,
+ ngx_http_upstream_rr_peer_t *oresolve,
+ ngx_http_upstream_rr_peers_t *opeers)
+{
+ in_port_t port;
+ ngx_str_t *server;
+ ngx_http_upstream_host_t *host;
+ ngx_http_upstream_rr_peer_t *peer, *template, *opeer, **peerp;
+
+ if (resolve == NULL || oresolve == NULL) {
+ return NGX_OK;
+ }
+
+ for (peerp = &peers->peer; *peerp; peerp = &(*peerp)->next) {
+ /* void */
+ }
+
+ ngx_http_upstream_rr_peers_rlock(opeers);
+
+ for (template = resolve; template; template = template->next) {
+ for (opeer = oresolve; opeer; opeer = opeer->next) {
+
+ if (opeer->host->name.len != template->host->name.len
+ || ngx_memcmp(opeer->host->name.data,
+ template->host->name.data,
+ template->host->name.len)
+ != 0)
+ {
+ continue;
+ }
+
+ if (opeer->host->service.len != template->host->service.len
+ || ngx_memcmp(opeer->host->service.data,
+ template->host->service.data,
+ template->host->service.len)
+ != 0)
+ {
+ continue;
+ }
+
+ host = opeer->host;
+
+ for (opeer = opeers->peer; opeer; opeer = opeer->next) {
+
+ if (opeer->host != host) {
+ continue;
+ }
+
+ peer = ngx_http_upstream_zone_copy_peer(peers, NULL);
+ if (peer == NULL) {
+ ngx_http_upstream_rr_peers_unlock(opeers);
+ return NGX_ERROR;
+ }
+
+ ngx_memcpy(peer->sockaddr, opeer->sockaddr, opeer->socklen);
+
+ if (template->host->service.len == 0) {
+ port = ngx_inet_get_port(template->sockaddr);
+ ngx_inet_set_port(peer->sockaddr, port);
+ }
+
+ peer->socklen = opeer->socklen;
+
+ peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen,
+ peer->name.data,
+ NGX_SOCKADDR_STRLEN, 1);
+
+ peer->host = template->host;
+
+ server = template->host->service.len ? &opeer->server
+ : &template->server;
+
+ peer->server.data = ngx_slab_alloc(peers->shpool, server->len);
+ if (peer->server.data == NULL) {
+ ngx_http_upstream_rr_peers_unlock(opeers);
+ return NGX_ERROR;
+ }
+
+ ngx_memcpy(peer->server.data, server->data, server->len);
+ peer->server.len = server->len;
+
+ if (host->service.len == 0) {
+ peer->weight = template->weight;
+
+ } else {
+ peer->weight = (template->weight != 1 ? template->weight
+ : opeer->weight);
+ }
+
+ peer->effective_weight = peer->weight;
+ peer->max_conns = template->max_conns;
+ peer->max_fails = template->max_fails;
+ peer->fail_timeout = template->fail_timeout;
+ peer->down = template->down;
+
+ (*peers->config)++;
+
+ *peerp = peer;
+ peerp = &peer->next;
+
+ peers->number++;
+ peers->tries += (peer->down == 0);
+ peers->total_weight += peer->weight;
+ peers->weighted = (peers->total_weight != peers->number);
+ }
+
+ break;
+ }
+ }
+
+ ngx_http_upstream_rr_peers_unlock(opeers);
+ return NGX_OK;
+}
+
+
static void
ngx_http_upstream_zone_set_single(ngx_http_upstream_srv_conf_t *uscf)
{
diff --git a/src/stream/ngx_stream_upstream_zone_module.c b/src/stream/ngx_stream_upstream_zone_module.c
index 0d6ab89f3..d92609feb 100644
--- a/src/stream/ngx_stream_upstream_zone_module.c
+++ b/src/stream/ngx_stream_upstream_zone_module.c
@@ -15,9 +15,15 @@ static char *ngx_stream_upstream_zone(ngx_conf_t *cf, ngx_command_t *cmd,
static ngx_int_t ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone,
void *data);
static ngx_stream_upstream_rr_peers_t *ngx_stream_upstream_zone_copy_peers(
- ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf);
+ ngx_slab_pool_t *shpool, ngx_stream_upstream_srv_conf_t *uscf,
+ ngx_stream_upstream_srv_conf_t *ouscf);
static ngx_stream_upstream_rr_peer_t *ngx_stream_upstream_zone_copy_peer(
ngx_stream_upstream_rr_peers_t *peers, ngx_stream_upstream_rr_peer_t *src);
+static ngx_int_t ngx_stream_upstream_zone_preresolve(
+ ngx_stream_upstream_rr_peer_t *resolve,
+ ngx_stream_upstream_rr_peers_t *peers,
+ ngx_stream_upstream_rr_peer_t *oresolve,
+ ngx_stream_upstream_rr_peers_t *opeers);
static void ngx_stream_upstream_zone_set_single(
ngx_stream_upstream_srv_conf_t *uscf);
static void ngx_stream_upstream_zone_remove_peer_locked(
@@ -125,11 +131,11 @@ static ngx_int_t
ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
{
size_t len;
- ngx_uint_t i;
+ ngx_uint_t i, j;
ngx_slab_pool_t *shpool;
ngx_stream_upstream_rr_peers_t *peers, **peersp;
- ngx_stream_upstream_srv_conf_t *uscf, **uscfp;
- ngx_stream_upstream_main_conf_t *umcf;
+ ngx_stream_upstream_srv_conf_t *uscf, *ouscf, **uscfp, **ouscfp;
+ ngx_stream_upstream_main_conf_t *umcf, *oumcf;

shpool = (ngx_slab_pool_t *) shm_zone->shm.addr;
umcf = shm_zone->data;
@@ -166,6 +172,7 @@ ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
/* copy peers to shared memory */

peersp = (ngx_stream_upstream_rr_peers_t **) (void *) &shpool->data;
+ oumcf = data;

for (i = 0; i < umcf->upstreams.nelts; i++) {
uscf = uscfp[i];
@@ -174,7 +181,38 @@ ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)
continue;
}

- peers = ngx_stream_upstream_zone_copy_peers(shpool, uscf);
+ ouscf = NULL;
+
+ if (oumcf) {
+ ouscfp = oumcf->upstreams.elts;
+
+ for (j = 0; j < oumcf->upstreams.nelts; j++) {
+
+ if (ouscfp[j]->shm_zone == NULL) {
+ continue;
+ }
+
+ if (ouscfp[j]->shm_zone->shm.name.len != shm_zone->shm.name.len
+ || ngx_memcmp(ouscfp[j]->shm_zone->shm.name.data,
+ shm_zone->shm.name.data,
+ shm_zone->shm.name.len)
+ != 0)
+ {
+ continue;
+ }
+
+ if (ouscfp[j]->host.len == uscf->host.len
+ && ngx_memcmp(ouscfp[j]->host.data, uscf->host.data,
+ uscf->host.len)
+ == 0)
+ {
+ ouscf = ouscfp[j];
+ break;
+ }
+ }
+ }
+
+ peers = ngx_stream_upstream_zone_copy_peers(shpool, uscf, ouscf);
if (peers == NULL) {
return NGX_ERROR;
}
@@ -189,12 +227,14 @@ ngx_stream_upstream_init_zone(ngx_shm_zone_t *shm_zone, void *data)

static ngx_stream_upstream_rr_peers_t *
ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
- ngx_stream_upstream_srv_conf_t *uscf)
+ ngx_stream_upstream_srv_conf_t *uscf, ngx_stream_upstream_srv_conf_t *ouscf)
{
ngx_str_t *name;
ngx_uint_t *config;
ngx_stream_upstream_rr_peer_t *peer, **peerp;
- ngx_stream_upstream_rr_peers_t *peers, *backup;
+ ngx_stream_upstream_rr_peers_t *peers, *opeers, *backup;
+
+ opeers = (ouscf ? ouscf->peer.data : NULL);

config = ngx_slab_calloc(shpool, sizeof(ngx_uint_t));
if (config == NULL) {
@@ -247,6 +287,16 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,
(*peers->config)++;
}

+ if (opeers) {
+
+ if (ngx_stream_upstream_zone_preresolve(peers->resolve, peers,
+ opeers->resolve, opeers)
+ != NGX_OK)
+ {
+ return NULL;
+ }
+ }
+
if (peers->next == NULL) {
goto done;
}
@@ -286,10 +336,30 @@ ngx_stream_upstream_zone_copy_peers(ngx_slab_pool_t *shpool,

peers->next = backup;

+ if (opeers && opeers->next) {
+
+ if (ngx_stream_upstream_zone_preresolve(peers->resolve, backup,
+ opeers->resolve, opeers->next)
+ != NGX_OK)
+ {
+ return NULL;
+ }
+
+ if (ngx_stream_upstream_zone_preresolve(backup->resolve, backup,
+ opeers->next->resolve,
+ opeers->next)
+ != NGX_OK)
+ {
+ return NULL;
+ }
+ }
+
done:

uscf->peer.data = peers;

+ ngx_stream_upstream_zone_set_single(uscf);
+
return peers;
}

@@ -401,6 +471,123 @@ failed:
}


+static ngx_int_t
+ngx_stream_upstream_zone_preresolve(ngx_stream_upstream_rr_peer_t *resolve,
+ ngx_stream_upstream_rr_peers_t *peers,
+ ngx_stream_upstream_rr_peer_t *oresolve,
+ ngx_stream_upstream_rr_peers_t *opeers)
+{
+ in_port_t port;
+ ngx_str_t *server;
+ ngx_stream_upstream_host_t *host;
+ ngx_stream_upstream_rr_peer_t *peer, *template, *opeer, **peerp;
+
+ if (resolve == NULL || oresolve == NULL) {
+ return NGX_OK;
+ }
+
+ for (peerp = &peers->peer; *peerp; peerp = &(*peerp)->next) {
+ /* void */
+ }
+
+ ngx_stream_upstream_rr_peers_rlock(opeers);
+
+ for (template = resolve; template; template = template->next) {
+ for (opeer = oresolve; opeer; opeer = opeer->next) {
+
+ if (opeer->host->name.len != template->host->name.len
+ || ngx_memcmp(opeer->host->name.data,
+ template->host->name.data,
+ template->host->name.len)
+ != 0)
+ {
+ continue;
+ }
+
+ if (opeer->host->service.len != template->host->service.len
+ || ngx_memcmp(opeer->host->service.data,
+ template->host->service.data,
+ template->host->service.len)
+ != 0)
+ {
+ continue;
+ }
+
+ host = opeer->host;
+
+ for (opeer = opeers->peer; opeer; opeer = opeer->next) {
+
+ if (opeer->host != host) {
+ continue;
+ }
+
+ peer = ngx_stream_upstream_zone_copy_peer(peers, NULL);
+ if (peer == NULL) {
+ ngx_stream_upstream_rr_peers_unlock(opeers);
+ return NGX_ERROR;
+ }
+
+ ngx_memcpy(peer->sockaddr, opeer->sockaddr, opeer->socklen);
+
+ if (template->host->service.len == 0) {
+ port = ngx_inet_get_port(template->sockaddr);
+ ngx_inet_set_port(peer->sockaddr, port);
+ }
+
+ peer->socklen = opeer->socklen;
+
+ peer->name.len = ngx_sock_ntop(peer->sockaddr, peer->socklen,
+ peer->name.data,
+ NGX_SOCKADDR_STRLEN, 1);
+
+ peer->host = template->host;
+
+ server = template->host->service.len ? &opeer->server
+ : &template->server;
+
+ peer->server.data = ngx_slab_alloc(peers->shpool, server->len);
+ if (peer->server.data == NULL) {
+ ngx_stream_upstream_rr_peers_unlock(opeers);
+ return NGX_ERROR;
+ }
+
+ ngx_memcpy(peer->server.data, server->data, server->len);
+ peer->server.len = server->len;
+
+ if (host->service.len == 0) {
+ peer->weight = template->weight;
+
+ } else {
+ peer->weight = (template->weight != 1 ? template->weight
+ : opeer->weight);
+ }
+
+ peer->effective_weight = peer->weight;
+ peer->max_conns = template->max_conns;
+ peer->max_fails = template->max_fails;
+ peer->fail_timeout = template->fail_timeout;
+ peer->down = template->down;
+
+ (*peers->config)++;
+
+ *peerp = peer;
+ peerp = &peer->next;
+
+ peers->number++;
+ peers->tries += (peer->down == 0);
+ peers->total_weight += peer->weight;
+ peers->weighted = (peers->total_weight != peers->number);
+ }
+
+ break;
+ }
+ }
+
+ ngx_stream_upstream_rr_peers_unlock(opeers);
+ return NGX_OK;
+}
+
+
static void
ngx_stream_upstream_zone_set_single(ngx_stream_upstream_srv_conf_t *uscf)
{
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel