re-sending the patch as an attachment as the formatting is still weird,
and fixed typo I spotted..

On 02/04/2023 18:57, J Carter wrote:
> Hello,
>
> I've also attached an example nginx.conf and test script that
> simulates the asynchronous close events.
> Two different test cases can be found within that, one with path /1
> for single peer upstream and /2 for multi-peer.
>
> You should see 2 upstream addresses repeated in a row
> per-upstream-server in the access log by default, as it fails
> through the cached connections & next performs next upstream tries.
>
> Any feedback would be appreciated.
>
> # HG changeset patch
> # User jordanc.carter@outlook.com
> # Date 1680457073 -3600
> #      Sun Apr 02 18:37:53 2023 +0100
> # Node ID 9ec4d7a8cdf6cdab00d09dff75fa6045f6f5533f
> # Parent  5f1d05a21287ba0290dd3a17ad501595b442a194
> Added keepalive_async_fails command to keepalive load balancer module.
> This value determines the number suspected keepalive race events
> per-upstream-try that will be tolerated before a subsequent network
> connection
> error is considered a true failure.
>
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6 src/event/ngx_event_connect.h
> --- a/src/event/ngx_event_connect.h    Tue Mar 28 18:01:54 2023 +0300
> +++ b/src/event/ngx_event_connect.h    Sun Apr 02 18:37:53 2023 +0100
> @@ -17,6 +17,7 @@
>  #define NGX_PEER_KEEPALIVE           1
>  #define NGX_PEER_NEXT                2
>  #define NGX_PEER_FAILED              4
> +#define NGX_PEER_ASYNC_FAILED        8
>
>
>  typedef struct ngx_peer_connection_s  ngx_peer_connection_t;
> @@ -41,6 +42,7 @@
>      ngx_str_t                       *name;
>
>      ngx_uint_t                       tries;
> +    ngx_uint_t                       async_fails;
>      ngx_msec_t                       start_time;
>
>      ngx_event_get_peer_pt            get;
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6
> src/http/modules/ngx_http_upstream_keepalive_module.c
> --- a/src/http/modules/ngx_http_upstream_keepalive_module.c    Tue Mar
> 28 18:01:54 2023 +0300
> +++ b/src/http/modules/ngx_http_upstream_keepalive_module.c    Sun Apr
> 02 18:37:53 2023 +0100
> @@ -13,6 +13,7 @@
>  typedef struct {
>      ngx_uint_t                         max_cached;
>      ngx_uint_t                         requests;
> +    ngx_uint_t                         max_async_fails;
>      ngx_msec_t                         time;
>      ngx_msec_t                         timeout;
>
> @@ -108,6 +109,13 @@
>        offsetof(ngx_http_upstream_keepalive_srv_conf_t, requests),
>        NULL },
>
> +     { ngx_string("keepalive_async_fails"),
> +      NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1,
> +      ngx_conf_set_num_slot,
> +      NGX_HTTP_SRV_CONF_OFFSET,
> +      offsetof(ngx_http_upstream_keepalive_srv_conf_t, max_async_fails),
> +      NULL },
> +
>        ngx_null_command
>  };
>
> @@ -160,6 +168,7 @@
>      ngx_conf_init_msec_value(kcf->time, 3600000);
>      ngx_conf_init_msec_value(kcf->timeout, 60000);
>      ngx_conf_init_uint_value(kcf->requests, 1000);
> +    ngx_conf_init_uint_value(kcf->max_async_fails, 2);
>
>      if (kcf->original_init_upstream(cf, us) != NGX_OK) {
>          return NGX_ERROR;
> @@ -320,6 +329,21 @@
>      u = kp->upstream;
>      c = pc->connection;
>
> +    if (state & NGX_PEER_ASYNC_FAILED) {
> +        pc->async_fails++;
> +
> +        if (pc->async_fails == 2) {
> +            pc->async_fails = 0;
> +            state = NGX_PEER_FAILED;
> +
> +        } else {
> +            pc->tries++;
> +        }
> +        goto invalid;
> +    }
> +
> +    pc->async_fails = 0;
> +
>      if (state & NGX_PEER_FAILED
>          || c == NULL
>          || c->read->eof
> @@ -529,6 +553,8 @@
>      conf->time = NGX_CONF_UNSET_MSEC;
>      conf->timeout = NGX_CONF_UNSET_MSEC;
>      conf->requests = NGX_CONF_UNSET_UINT;
> +    conf->max_async_fails = NGX_CONF_UNSET_UINT;
> +
>
>      return conf;
>  }
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6 src/http/ngx_http_upstream.c
> --- a/src/http/ngx_http_upstream.c    Tue Mar 28 18:01:54 2023 +0300
> +++ b/src/http/ngx_http_upstream.c    Sun Apr 02 18:37:53 2023 +0100
> @@ -4317,6 +4317,8 @@
>          {
>              state = NGX_PEER_NEXT;
>
> +        } else if (u->peer.cached && ft_type ==
> NGX_HTTP_UPSTREAM_FT_ERROR) {
> +            state = NGX_PEER_ASYNC_FAILED;
>          } else {
>              state = NGX_PEER_FAILED;
>          }
> @@ -4330,11 +4332,6 @@
>                        "upstream timed out");
>      }
>
> -    if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
> -        /* TODO: inform balancer instead */
> -        u->peer.tries++;
> -    }
> -
>      switch (ft_type) {
>
>      case NGX_HTTP_UPSTREAM_FT_TIMEOUT:
> @@ -4421,7 +4418,6 @@
>              return;
>          }
>  #endif
> -
>          ngx_http_upstream_finalize_request(r, u, status);
>          return;
>      }
> diff -r 5f1d05a21287 -r 9ec4d7a8cdf6
> src/http/ngx_http_upstream_round_robin.c
> --- a/src/http/ngx_http_upstream_round_robin.c    Tue Mar 28 18:01:54
> 2023 +0300
> +++ b/src/http/ngx_http_upstream_round_robin.c    Sun Apr 02 18:37:53
> 2023 +0100
> @@ -297,6 +297,7 @@
>      r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
>      r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
>      r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
> +    r->upstream->peer.async_fails = 0;
>  #if (NGX_HTTP_SSL)
>      r->upstream->peer.set_session =
> ngx_http_upstream_set_round_robin_peer_session;
> @@ -418,6 +419,7 @@
>      r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
>      r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
>      r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
> +    r->upstream->peer.async_fails = 0;
>  #if (NGX_HTTP_SSL)
>      r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
>      r->upstream->peer.save_session =
> ngx_http_upstream_empty_save_session;
> @@ -459,7 +461,10 @@
>
>          rrp->current = peer;
>
> -    } else {
> +    } else if (pc->async_fails > 0) {
> +        peer = rrp->current;
> +    }
> +    else {
>
>          /* there are several peers */
>
> @@ -615,18 +620,7 @@
>      ngx_http_upstream_rr_peers_rlock(rrp->peers);
>      ngx_http_upstream_rr_peer_lock(rrp->peers, peer);
>
> -    if (rrp->peers->single) {
> -
> -        peer->conns--;
> -
> -        ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
> -        ngx_http_upstream_rr_peers_unlock(rrp->peers);
> -
> -        pc->tries = 0;
> -        return;
> -    }
> -
> -    if (state & NGX_PEER_FAILED) {
> +    if (state & NGX_PEER_FAILED && !rrp->peers->single) {
>          now = ngx_time();
>
>          peer->fails++;
>
> _______________________________________________
> nginx-devel mailing list
> nginx-devel@nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx-devel
# HG changeset patch
# User jordanc.carter@outlook.com
# Date 1680459126 -3600
# Sun Apr 02 19:12:06 2023 +0100
# Node ID 4295bf4613e155e063914ac10f898dbe98ae4a54
# Parent 5f1d05a21287ba0290dd3a17ad501595b442a194
Added keepalive_async_fails command to keepalive load balancer module.
This value determines the number suspected keepalive race events
per-upstream-try that will be tolerated before a subsequent network connection
error is considered a true failure.

diff -r 5f1d05a21287 -r 4295bf4613e1 src/event/ngx_event_connect.h
--- a/src/event/ngx_event_connect.h Tue Mar 28 18:01:54 2023 +0300
+++ b/src/event/ngx_event_connect.h Sun Apr 02 19:12:06 2023 +0100
@@ -17,6 +17,7 @@
#define NGX_PEER_KEEPALIVE 1
#define NGX_PEER_NEXT 2
#define NGX_PEER_FAILED 4
+#define NGX_PEER_ASYNC_FAILED 8


typedef struct ngx_peer_connection_s ngx_peer_connection_t;
@@ -41,6 +42,7 @@
ngx_str_t *name;

ngx_uint_t tries;
+ ngx_uint_t async_fails;
ngx_msec_t start_time;

ngx_event_get_peer_pt get;
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/modules/ngx_http_upstream_keepalive_module.c
--- a/src/http/modules/ngx_http_upstream_keepalive_module.c Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/modules/ngx_http_upstream_keepalive_module.c Sun Apr 02 19:12:06 2023 +0100
@@ -13,6 +13,7 @@
typedef struct {
ngx_uint_t max_cached;
ngx_uint_t requests;
+ ngx_uint_t max_async_fails;
ngx_msec_t time;
ngx_msec_t timeout;

@@ -108,6 +109,13 @@
offsetof(ngx_http_upstream_keepalive_srv_conf_t, requests),
NULL },

+ { ngx_string("keepalive_async_fails"),
+ NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_num_slot,
+ NGX_HTTP_SRV_CONF_OFFSET,
+ offsetof(ngx_http_upstream_keepalive_srv_conf_t, max_async_fails),
+ NULL },
+
ngx_null_command
};

@@ -160,6 +168,7 @@
ngx_conf_init_msec_value(kcf->time, 3600000);
ngx_conf_init_msec_value(kcf->timeout, 60000);
ngx_conf_init_uint_value(kcf->requests, 1000);
+ ngx_conf_init_uint_value(kcf->max_async_fails, 2);

if (kcf->original_init_upstream(cf, us) != NGX_OK) {
return NGX_ERROR;
@@ -320,6 +329,21 @@
u = kp->upstream;
c = pc->connection;

+ if (state & NGX_PEER_ASYNC_FAILED) {
+ pc->async_fails++;
+
+ if (pc->async_fails == kp->conf->max_async_fails) {
+ pc->async_fails = 0;
+ state = NGX_PEER_FAILED;
+
+ } else {
+ pc->tries++;
+ }
+ goto invalid;
+ }
+
+ pc->async_fails = 0;
+
if (state & NGX_PEER_FAILED
|| c == NULL
|| c->read->eof
@@ -529,6 +553,8 @@
conf->time = NGX_CONF_UNSET_MSEC;
conf->timeout = NGX_CONF_UNSET_MSEC;
conf->requests = NGX_CONF_UNSET_UINT;
+ conf->max_async_fails = NGX_CONF_UNSET_UINT;
+

return conf;
}
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/ngx_http_upstream.c Sun Apr 02 19:12:06 2023 +0100
@@ -4317,6 +4317,8 @@
{
state = NGX_PEER_NEXT;

+ } else if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
+ state = NGX_PEER_ASYNC_FAILED;
} else {
state = NGX_PEER_FAILED;
}
@@ -4330,11 +4332,6 @@
"upstream timed out");
}

- if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
- /* TODO: inform balancer instead */
- u->peer.tries++;
- }
-
switch (ft_type) {

case NGX_HTTP_UPSTREAM_FT_TIMEOUT:
@@ -4421,7 +4418,6 @@
return;
}
#endif
-
ngx_http_upstream_finalize_request(r, u, status);
return;
}
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/ngx_http_upstream_round_robin.c
--- a/src/http/ngx_http_upstream_round_robin.c Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/ngx_http_upstream_round_robin.c Sun Apr 02 19:12:06 2023 +0100
@@ -297,6 +297,7 @@
r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
+ r->upstream->peer.async_fails = 0;
#if (NGX_HTTP_SSL)
r->upstream->peer.set_session =
ngx_http_upstream_set_round_robin_peer_session;
@@ -418,6 +419,7 @@
r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
+ r->upstream->peer.async_fails = 0;
#if (NGX_HTTP_SSL)
r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
r->upstream->peer.save_session = ngx_http_upstream_empty_save_session;
@@ -459,7 +461,10 @@

rrp->current = peer;

- } else {
+ } else if (pc->async_fails > 0) {
+ peer = rrp->current;
+ }
+ else {

/* there are several peers */

@@ -615,18 +620,7 @@
ngx_http_upstream_rr_peers_rlock(rrp->peers);
ngx_http_upstream_rr_peer_lock(rrp->peers, peer);

- if (rrp->peers->single) {
-
- peer->conns--;
-
- ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
- ngx_http_upstream_rr_peers_unlock(rrp->peers);
-
- pc->tries = 0;
- return;
- }
-
- if (state & NGX_PEER_FAILED) {
+ if (state & NGX_PEER_FAILED && !rrp->peers->single) {
now = ngx_time();

peer->fails++;
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel