# HG changeset patch
# User Roman Arutyunyan <arut@nginx.com>
# Date 1673868053 0
# Mon Jan 16 11:20:53 2023 +0000
# Branch quic
# Node ID 7873810feab00761677f02224a2f6c1b0f130203
# Parent 52681b76d4db0e9e814f1e69b0d6b35f5ac630a2
QUIC: client sockets.

In client sockets mode, a new socket is created for each client. This socket
is bound to server address and connected to client address. This allows for
seamless configuration reload and binary upgrade.

This mode is enabled by default and can be disabled by
"--without-quic_client_sockets" configure option.

With this approach, it is possible that some new connection packets will arrive
not to the listen socket, but to a client socket due to a race between bind()
and connect(). Such packets initiate new connections in the workers they end
up in.

diff --git a/auto/modules b/auto/modules
--- a/auto/modules
+++ b/auto/modules
@@ -1378,6 +1378,10 @@ if [ $USE_OPENSSL_QUIC = YES ]; then

have=NGX_QUIC_BPF . auto/have
fi
+
+ if [ $QUIC_CLIENT_SOCKETS = YES ]; then
+ have=NGX_QUIC_CLIENT_SOCKETS . auto/have
+ fi
fi


diff --git a/auto/options b/auto/options
--- a/auto/options
+++ b/auto/options
@@ -45,6 +45,7 @@ USE_THREADS=NO

NGX_FILE_AIO=NO

+QUIC_CLIENT_SOCKETS=YES
QUIC_BPF=NO

HTTP=YES
@@ -216,6 +217,7 @@ do

--with-file-aio) NGX_FILE_AIO=YES ;;

+ --without-quic_client_sockets) QUIC_CLIENT_SOCKETS=NONE ;;
--without-quic_bpf_module) QUIC_BPF=NONE ;;

--with-ipv6)
@@ -452,6 +454,7 @@ cat << END

--with-file-aio enable file AIO support

+ --without-quic_client_sockets disable QUIC client sockets
--without-quic_bpf_module disable ngx_quic_bpf_module

--with-http_ssl_module enable ngx_http_ssl_module
diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -513,6 +513,33 @@ ngx_open_listening_sockets(ngx_cycle_t *

#if (NGX_HAVE_REUSEPORT)

+#if (NGX_QUIC_CLIENT_SOCKETS && defined SO_REUSEPORT_LB)
+
+ if (ls[i].quic) {
+ int reuseport;
+
+ reuseport = 1;
+
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &reuseport, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+ "setsockopt(SO_REUSEPORT) %V failed",
+ &ls[i].addr_text);
+
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+ ngx_close_socket_n " %V failed",
+ &ls[i].addr_text);
+ }
+
+ return NGX_ERROR;
+ }
+ }
+
+#endif
+
if (ls[i].reuseport && !ngx_test_config) {
int reuseport;

diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
--- a/src/event/quic/ngx_event_quic.c
+++ b/src/event/quic/ngx_event_quic.c
@@ -10,11 +10,15 @@
#include <ngx_event_quic_connection.h>


+#define NGX_QUIC_LINGERING_TIMEOUT 50
+
+
static ngx_quic_connection_t *ngx_quic_new_connection(ngx_connection_t *c,
ngx_quic_conf_t *conf, ngx_quic_header_t *pkt);
static ngx_int_t ngx_quic_handle_stateless_reset(ngx_connection_t *c,
ngx_quic_header_t *pkt);
static void ngx_quic_input_handler(ngx_event_t *rev);
+static void ngx_quic_lingering_handler(ngx_event_t *rev);
static void ngx_quic_close_handler(ngx_event_t *ev);

static ngx_int_t ngx_quic_handle_packet(ngx_connection_t *c,
@@ -208,6 +212,13 @@ ngx_quic_run(ngx_connection_t *c, ngx_qu
ngx_quic_close_connection(c, rc);
return;
}
+
+ if (!c->shared) {
+ if (ngx_add_event(c->read, NGX_READ_EVENT, 0) == NGX_ERROR) {
+ ngx_quic_close_connection(c, NGX_ERROR);
+ return;
+ }
+ }
}


@@ -432,6 +443,10 @@ ngx_quic_input_handler(ngx_event_t *rev)

return;
}
+
+ if (!c->shared) {
+ ngx_quic_recvmsg(rev);
+ }
}


@@ -574,6 +589,15 @@ quic_done:

c->destroyed = 1;

+ if (c->read->ready) {
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic lingering");
+
+ c->read->handler = ngx_quic_lingering_handler;
+ ngx_add_timer(c->read, NGX_QUIC_LINGERING_TIMEOUT);
+
+ return;
+ }
+
pool = c->pool;

ngx_close_connection(c);
@@ -582,6 +606,31 @@ quic_done:
}


+static void
+ngx_quic_lingering_handler(ngx_event_t *rev)
+{
+ ngx_pool_t *pool;
+ ngx_connection_t *c;
+
+ c = rev->data;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic lingering handler");
+
+ if (rev->ready && !rev->timedout) {
+ ngx_quic_recvmsg(rev);
+ }
+
+ if (!rev->ready || rev->timedout) {
+ pool = c->pool;
+
+ ngx_close_connection(c);
+ ngx_destroy_pool(pool);
+
+ return;
+ }
+}
+
+
void
ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err,
const char *reason)
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -437,8 +437,10 @@ ngx_quic_send_segments(ngx_connection_t
msg.msg_iov = &iov;
msg.msg_iovlen = 1;

- msg.msg_name = sockaddr;
- msg.msg_namelen = socklen;
+ if (c->shared) {
+ msg.msg_name = sockaddr;
+ msg.msg_namelen = socklen;
+ }

msg.msg_control = msg_control;
msg.msg_controllen = sizeof(msg_control);
@@ -455,7 +457,7 @@ ngx_quic_send_segments(ngx_connection_t
*valp = segment;

#if (NGX_HAVE_ADDRINFO_CMSG)
- if (c->listening && c->listening->wildcard && c->local_sockaddr) {
+ if (c->shared && c->listening->wildcard) {
cmsg = CMSG_NXTHDR(&msg, cmsg);
clen += ngx_set_srcaddr_cmsg(cmsg, c->local_sockaddr);
}
@@ -747,11 +749,13 @@ ngx_quic_send(ngx_connection_t *c, u_cha
msg.msg_iov = &iov;
msg.msg_iovlen = 1;

- msg.msg_name = sockaddr;
- msg.msg_namelen = socklen;
+ if (c->shared) {
+ msg.msg_name = sockaddr;
+ msg.msg_namelen = socklen;
+ }

#if (NGX_HAVE_ADDRINFO_CMSG)
- if (c->listening && c->listening->wildcard && c->local_sockaddr) {
+ if (c->shared && c->listening->wildcard) {

msg.msg_control = msg_control;
msg.msg_controllen = sizeof(msg_control);
diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c
--- a/src/event/quic/ngx_event_quic_udp.c
+++ b/src/event/quic/ngx_event_quic_udp.c
@@ -11,6 +11,11 @@
#include <ngx_event_quic_connection.h>


+#if (NGX_QUIC_CLIENT_SOCKETS)
+static ngx_int_t ngx_quic_create_client_socket_connection(ngx_connection_t *lc,
+ ngx_connection_t **pc, struct sockaddr *sockaddr, socklen_t socklen,
+ struct sockaddr *local_sockaddr, socklen_t local_socklen);
+#endif
static void ngx_quic_close_accepted_connection(ngx_connection_t *c);
static ngx_connection_t *ngx_quic_lookup_connection(ngx_listening_t *ls,
ngx_str_t *key, struct sockaddr *local_sockaddr, socklen_t local_socklen);
@@ -48,7 +53,6 @@ ngx_quic_recvmsg(ngx_event_t *ev)

lc = ev->data;
ls = lc->listening;
- ev->ready = 0;

ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"quic recvmsg on %V, ready: %d",
@@ -65,8 +69,17 @@ ngx_quic_recvmsg(ngx_event_t *ev)
msg.msg_iov = iov;
msg.msg_iovlen = 1;

+ if (lc->local_sockaddr) {
+ local_sockaddr = lc->local_sockaddr;
+ local_socklen = lc->local_socklen;
+
+ } else {
+ local_sockaddr = ls->sockaddr;
+ local_socklen = ls->socklen;
+ }
+
#if (NGX_HAVE_ADDRINFO_CMSG)
- if (ls->wildcard) {
+ if (ngx_inet_wildcard(local_sockaddr)) {
msg.msg_control = &msg_control;
msg.msg_controllen = sizeof(msg_control);

@@ -82,6 +95,7 @@ ngx_quic_recvmsg(ngx_event_t *ev)
if (err == NGX_EAGAIN) {
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err,
"quic recvmsg() not ready");
+ ev->ready = 0;
return;
}

@@ -121,12 +135,9 @@ ngx_quic_recvmsg(ngx_event_t *ev)

#endif

- local_sockaddr = ls->sockaddr;
- local_socklen = ls->socklen;
-
#if (NGX_HAVE_ADDRINFO_CMSG)

- if (ls->wildcard) {
+ if (ngx_inet_wildcard(local_sockaddr)) {
struct cmsghdr *cmsg;

ngx_memcpy(&lsa, local_sockaddr, local_socklen);
@@ -201,6 +212,17 @@ ngx_quic_recvmsg(ngx_event_t *ev)
}
#endif

+#if (NGX_QUIC_CLIENT_SOCKETS)
+ if (c == NULL) {
+ if (ngx_quic_create_client_socket_connection(lc, &c,
+ sockaddr, socklen, local_sockaddr, local_socklen)
+ != NGX_OK)
+ {
+ return;
+ }
+ }
+#endif
+
if (c == NULL) {
c = ngx_get_connection(lc->fd, ev->log);
if (c == NULL) {
@@ -243,17 +265,13 @@ ngx_quic_recvmsg(ngx_event_t *ev)
c->pool->log = log;
c->listening = ls;

- if (local_sockaddr == &lsa.sockaddr) {
- local_sockaddr = ngx_palloc(c->pool, local_socklen);
- if (local_sockaddr == NULL) {
- ngx_quic_close_accepted_connection(c);
- return;
- }
-
- ngx_memcpy(local_sockaddr, &lsa, local_socklen);
+ c->local_sockaddr = ngx_palloc(c->pool, local_socklen);
+ if (c->local_sockaddr == NULL) {
+ ngx_quic_close_accepted_connection(c);
+ return;
}

- c->local_sockaddr = local_sockaddr;
+ ngx_memcpy(c->local_sockaddr, local_sockaddr, local_socklen);
c->local_socklen = local_socklen;

c->buffer = ngx_create_temp_buf(c->pool, n);
@@ -267,7 +285,7 @@ ngx_quic_recvmsg(ngx_event_t *ev)
rev = c->read;
wev = c->write;

- rev->active = 1;
+ rev->ready = (c->shared ? 0 : 1);
wev->ready = 1;

rev->log = log;
@@ -341,6 +359,94 @@ ngx_quic_recvmsg(ngx_event_t *ev)
}


+#if (NGX_QUIC_CLIENT_SOCKETS)
+
+static ngx_int_t
+ngx_quic_create_client_socket_connection(ngx_connection_t *lc,
+ ngx_connection_t **pc, struct sockaddr *sockaddr, socklen_t socklen,
+ struct sockaddr *local_sockaddr, socklen_t local_socklen)
+{
+ int value;
+ ngx_socket_t s;
+
+ s = ngx_socket(sockaddr->sa_family, SOCK_DGRAM, 0);
+ if (s == (ngx_socket_t) -1) {
+ ngx_log_error(NGX_LOG_ERR, lc->log, ngx_socket_errno,
+ ngx_socket_n " failed");
+ return NGX_ERROR;
+ }
+
+ if (ngx_nonblocking(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ ngx_nonblocking_n " client socket failed");
+ goto failed;
+ }
+
+ value = 1;
+
+#if (NGX_HAVE_REUSEPORT && !NGX_LINUX)
+
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &value, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "setsockopt(SO_REUSEPORT) client socket failed");
+ goto failed;
+ }
+
+#else
+
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR,
+ (const void *) &value, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "setsockopt(SO_REUSEADDR) client socket failed");
+ goto failed;
+ }
+
+#endif
+
+ if (bind(s, local_sockaddr, local_socklen) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "bind() for client socket failed");
+ goto failed;
+ }
+
+ if (connect(s, sockaddr, socklen) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "connect() to client failed");
+ goto failed;
+ }
+
+ *pc = ngx_get_connection(s, lc->log);
+ if (*pc == NULL) {
+ goto failed;
+ }
+
+ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, lc->log, 0,
+ "quic client socket connection fd:%d r:%d",
+ s, lc->listening->connection != lc);
+
+ ngx_accept_disabled = ngx_cycle->connection_n / 8
+ - ngx_cycle->free_connection_n;
+
+ return NGX_OK;
+
+failed:
+
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ ngx_close_socket_n " client failed");
+ }
+
+ return NGX_ERROR;
+}
+
+#endif
+
+
static void
ngx_quic_close_accepted_connection(ngx_connection_t *c)
{
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel