Welcome! Log In Create A New Profile

Advanced

[PATCH 4 of 4] QUIC: client sockets

Roman Arutyunyan
January 17, 2023 06:26AM
# HG changeset patch
# User Roman Arutyunyan <arut@nginx.com>
# Date 1673868053 0
# Mon Jan 16 11:20:53 2023 +0000
# Branch quic
# Node ID 7873810feab00761677f02224a2f6c1b0f130203
# Parent 52681b76d4db0e9e814f1e69b0d6b35f5ac630a2
QUIC: client sockets.

In client sockets mode, a new socket is created for each client. This socket
is bound to server address and connected to client address. This allows for
seamless configuration reload and binary upgrade.

This mode is enabled by default and can be disabled by
"--without-quic_client_sockets" configure option.

With this approach, it is possible that some new connection packets will arrive
not to the listen socket, but to a client socket due to a race between bind()
and connect(). Such packets initiate new connections in the workers they end
up in.

diff --git a/auto/modules b/auto/modules
--- a/auto/modules
+++ b/auto/modules
@@ -1378,6 +1378,10 @@ if [ $USE_OPENSSL_QUIC = YES ]; then

have=NGX_QUIC_BPF . auto/have
fi
+
+ if [ $QUIC_CLIENT_SOCKETS = YES ]; then
+ have=NGX_QUIC_CLIENT_SOCKETS . auto/have
+ fi
fi


diff --git a/auto/options b/auto/options
--- a/auto/options
+++ b/auto/options
@@ -45,6 +45,7 @@ USE_THREADS=NO

NGX_FILE_AIO=NO

+QUIC_CLIENT_SOCKETS=YES
QUIC_BPF=NO

HTTP=YES
@@ -216,6 +217,7 @@ do

--with-file-aio) NGX_FILE_AIO=YES ;;

+ --without-quic_client_sockets) QUIC_CLIENT_SOCKETS=NONE ;;
--without-quic_bpf_module) QUIC_BPF=NONE ;;

--with-ipv6)
@@ -452,6 +454,7 @@ cat << END

--with-file-aio enable file AIO support

+ --without-quic_client_sockets disable QUIC client sockets
--without-quic_bpf_module disable ngx_quic_bpf_module

--with-http_ssl_module enable ngx_http_ssl_module
diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -513,6 +513,33 @@ ngx_open_listening_sockets(ngx_cycle_t *

#if (NGX_HAVE_REUSEPORT)

+#if (NGX_QUIC_CLIENT_SOCKETS && defined SO_REUSEPORT_LB)
+
+ if (ls[i].quic) {
+ int reuseport;
+
+ reuseport = 1;
+
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &reuseport, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+ "setsockopt(SO_REUSEPORT) %V failed",
+ &ls[i].addr_text);
+
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+ ngx_close_socket_n " %V failed",
+ &ls[i].addr_text);
+ }
+
+ return NGX_ERROR;
+ }
+ }
+
+#endif
+
if (ls[i].reuseport && !ngx_test_config) {
int reuseport;

diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
--- a/src/event/quic/ngx_event_quic.c
+++ b/src/event/quic/ngx_event_quic.c
@@ -10,11 +10,15 @@
#include <ngx_event_quic_connection.h>


+#define NGX_QUIC_LINGERING_TIMEOUT 50
+
+
static ngx_quic_connection_t *ngx_quic_new_connection(ngx_connection_t *c,
ngx_quic_conf_t *conf, ngx_quic_header_t *pkt);
static ngx_int_t ngx_quic_handle_stateless_reset(ngx_connection_t *c,
ngx_quic_header_t *pkt);
static void ngx_quic_input_handler(ngx_event_t *rev);
+static void ngx_quic_lingering_handler(ngx_event_t *rev);
static void ngx_quic_close_handler(ngx_event_t *ev);

static ngx_int_t ngx_quic_handle_packet(ngx_connection_t *c,
@@ -208,6 +212,13 @@ ngx_quic_run(ngx_connection_t *c, ngx_qu
ngx_quic_close_connection(c, rc);
return;
}
+
+ if (!c->shared) {
+ if (ngx_add_event(c->read, NGX_READ_EVENT, 0) == NGX_ERROR) {
+ ngx_quic_close_connection(c, NGX_ERROR);
+ return;
+ }
+ }
}


@@ -432,6 +443,10 @@ ngx_quic_input_handler(ngx_event_t *rev)

return;
}
+
+ if (!c->shared) {
+ ngx_quic_recvmsg(rev);
+ }
}


@@ -574,6 +589,15 @@ quic_done:

c->destroyed = 1;

+ if (c->read->ready) {
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic lingering");
+
+ c->read->handler = ngx_quic_lingering_handler;
+ ngx_add_timer(c->read, NGX_QUIC_LINGERING_TIMEOUT);
+
+ return;
+ }
+
pool = c->pool;

ngx_close_connection(c);
@@ -582,6 +606,31 @@ quic_done:
}


+static void
+ngx_quic_lingering_handler(ngx_event_t *rev)
+{
+ ngx_pool_t *pool;
+ ngx_connection_t *c;
+
+ c = rev->data;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, "quic lingering handler");
+
+ if (rev->ready && !rev->timedout) {
+ ngx_quic_recvmsg(rev);
+ }
+
+ if (!rev->ready || rev->timedout) {
+ pool = c->pool;
+
+ ngx_close_connection(c);
+ ngx_destroy_pool(pool);
+
+ return;
+ }
+}
+
+
void
ngx_quic_finalize_connection(ngx_connection_t *c, ngx_uint_t err,
const char *reason)
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -437,8 +437,10 @@ ngx_quic_send_segments(ngx_connection_t
msg.msg_iov = &iov;
msg.msg_iovlen = 1;

- msg.msg_name = sockaddr;
- msg.msg_namelen = socklen;
+ if (c->shared) {
+ msg.msg_name = sockaddr;
+ msg.msg_namelen = socklen;
+ }

msg.msg_control = msg_control;
msg.msg_controllen = sizeof(msg_control);
@@ -455,7 +457,7 @@ ngx_quic_send_segments(ngx_connection_t
*valp = segment;

#if (NGX_HAVE_ADDRINFO_CMSG)
- if (c->listening && c->listening->wildcard && c->local_sockaddr) {
+ if (c->shared && c->listening->wildcard) {
cmsg = CMSG_NXTHDR(&msg, cmsg);
clen += ngx_set_srcaddr_cmsg(cmsg, c->local_sockaddr);
}
@@ -747,11 +749,13 @@ ngx_quic_send(ngx_connection_t *c, u_cha
msg.msg_iov = &iov;
msg.msg_iovlen = 1;

- msg.msg_name = sockaddr;
- msg.msg_namelen = socklen;
+ if (c->shared) {
+ msg.msg_name = sockaddr;
+ msg.msg_namelen = socklen;
+ }

#if (NGX_HAVE_ADDRINFO_CMSG)
- if (c->listening && c->listening->wildcard && c->local_sockaddr) {
+ if (c->shared && c->listening->wildcard) {

msg.msg_control = msg_control;
msg.msg_controllen = sizeof(msg_control);
diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c
--- a/src/event/quic/ngx_event_quic_udp.c
+++ b/src/event/quic/ngx_event_quic_udp.c
@@ -11,6 +11,11 @@
#include <ngx_event_quic_connection.h>


+#if (NGX_QUIC_CLIENT_SOCKETS)
+static ngx_int_t ngx_quic_create_client_socket_connection(ngx_connection_t *lc,
+ ngx_connection_t **pc, struct sockaddr *sockaddr, socklen_t socklen,
+ struct sockaddr *local_sockaddr, socklen_t local_socklen);
+#endif
static void ngx_quic_close_accepted_connection(ngx_connection_t *c);
static ngx_connection_t *ngx_quic_lookup_connection(ngx_listening_t *ls,
ngx_str_t *key, struct sockaddr *local_sockaddr, socklen_t local_socklen);
@@ -48,7 +53,6 @@ ngx_quic_recvmsg(ngx_event_t *ev)

lc = ev->data;
ls = lc->listening;
- ev->ready = 0;

ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"quic recvmsg on %V, ready: %d",
@@ -65,8 +69,17 @@ ngx_quic_recvmsg(ngx_event_t *ev)
msg.msg_iov = iov;
msg.msg_iovlen = 1;

+ if (lc->local_sockaddr) {
+ local_sockaddr = lc->local_sockaddr;
+ local_socklen = lc->local_socklen;
+
+ } else {
+ local_sockaddr = ls->sockaddr;
+ local_socklen = ls->socklen;
+ }
+
#if (NGX_HAVE_ADDRINFO_CMSG)
- if (ls->wildcard) {
+ if (ngx_inet_wildcard(local_sockaddr)) {
msg.msg_control = &msg_control;
msg.msg_controllen = sizeof(msg_control);

@@ -82,6 +95,7 @@ ngx_quic_recvmsg(ngx_event_t *ev)
if (err == NGX_EAGAIN) {
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err,
"quic recvmsg() not ready");
+ ev->ready = 0;
return;
}

@@ -121,12 +135,9 @@ ngx_quic_recvmsg(ngx_event_t *ev)

#endif

- local_sockaddr = ls->sockaddr;
- local_socklen = ls->socklen;
-
#if (NGX_HAVE_ADDRINFO_CMSG)

- if (ls->wildcard) {
+ if (ngx_inet_wildcard(local_sockaddr)) {
struct cmsghdr *cmsg;

ngx_memcpy(&lsa, local_sockaddr, local_socklen);
@@ -201,6 +212,17 @@ ngx_quic_recvmsg(ngx_event_t *ev)
}
#endif

+#if (NGX_QUIC_CLIENT_SOCKETS)
+ if (c == NULL) {
+ if (ngx_quic_create_client_socket_connection(lc, &c,
+ sockaddr, socklen, local_sockaddr, local_socklen)
+ != NGX_OK)
+ {
+ return;
+ }
+ }
+#endif
+
if (c == NULL) {
c = ngx_get_connection(lc->fd, ev->log);
if (c == NULL) {
@@ -243,17 +265,13 @@ ngx_quic_recvmsg(ngx_event_t *ev)
c->pool->log = log;
c->listening = ls;

- if (local_sockaddr == &lsa.sockaddr) {
- local_sockaddr = ngx_palloc(c->pool, local_socklen);
- if (local_sockaddr == NULL) {
- ngx_quic_close_accepted_connection(c);
- return;
- }
-
- ngx_memcpy(local_sockaddr, &lsa, local_socklen);
+ c->local_sockaddr = ngx_palloc(c->pool, local_socklen);
+ if (c->local_sockaddr == NULL) {
+ ngx_quic_close_accepted_connection(c);
+ return;
}

- c->local_sockaddr = local_sockaddr;
+ ngx_memcpy(c->local_sockaddr, local_sockaddr, local_socklen);
c->local_socklen = local_socklen;

c->buffer = ngx_create_temp_buf(c->pool, n);
@@ -267,7 +285,7 @@ ngx_quic_recvmsg(ngx_event_t *ev)
rev = c->read;
wev = c->write;

- rev->active = 1;
+ rev->ready = (c->shared ? 0 : 1);
wev->ready = 1;

rev->log = log;
@@ -341,6 +359,94 @@ ngx_quic_recvmsg(ngx_event_t *ev)
}


+#if (NGX_QUIC_CLIENT_SOCKETS)
+
+static ngx_int_t
+ngx_quic_create_client_socket_connection(ngx_connection_t *lc,
+ ngx_connection_t **pc, struct sockaddr *sockaddr, socklen_t socklen,
+ struct sockaddr *local_sockaddr, socklen_t local_socklen)
+{
+ int value;
+ ngx_socket_t s;
+
+ s = ngx_socket(sockaddr->sa_family, SOCK_DGRAM, 0);
+ if (s == (ngx_socket_t) -1) {
+ ngx_log_error(NGX_LOG_ERR, lc->log, ngx_socket_errno,
+ ngx_socket_n " failed");
+ return NGX_ERROR;
+ }
+
+ if (ngx_nonblocking(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ ngx_nonblocking_n " client socket failed");
+ goto failed;
+ }
+
+ value = 1;
+
+#if (NGX_HAVE_REUSEPORT && !NGX_LINUX)
+
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &value, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "setsockopt(SO_REUSEPORT) client socket failed");
+ goto failed;
+ }
+
+#else
+
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR,
+ (const void *) &value, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "setsockopt(SO_REUSEADDR) client socket failed");
+ goto failed;
+ }
+
+#endif
+
+ if (bind(s, local_sockaddr, local_socklen) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "bind() for client socket failed");
+ goto failed;
+ }
+
+ if (connect(s, sockaddr, socklen) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ "connect() to client failed");
+ goto failed;
+ }
+
+ *pc = ngx_get_connection(s, lc->log);
+ if (*pc == NULL) {
+ goto failed;
+ }
+
+ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, lc->log, 0,
+ "quic client socket connection fd:%d r:%d",
+ s, lc->listening->connection != lc);
+
+ ngx_accept_disabled = ngx_cycle->connection_n / 8
+ - ngx_cycle->free_connection_n;
+
+ return NGX_OK;
+
+failed:
+
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, lc->log, ngx_socket_errno,
+ ngx_close_socket_n " client failed");
+ }
+
+ return NGX_ERROR;
+}
+
+#endif
+
+
static void
ngx_quic_close_accepted_connection(ngx_connection_t *c)
{
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel
Subject Author Views Posted

[PATCH 0 of 4] QUIC packet routing improvements

Roman Arutyunyan 390 January 17, 2023 06:26AM

[PATCH 1 of 4] QUIC: handle datagrams directly in ngx_quic_recvmsg()

Roman Arutyunyan 173 January 17, 2023 06:26AM

[PATCH 2 of 4] QUIC: eliminated timeout handling in listen connection read event

Roman Arutyunyan 116 January 17, 2023 06:26AM

[PATCH 3 of 4] QUIC: eBPF worker sockets

Roman Arutyunyan 207 January 17, 2023 06:26AM

[PATCH 4 of 4] QUIC: client sockets

Roman Arutyunyan 120 January 17, 2023 06:26AM



Sorry, you do not have permission to post/reply in this forum.

Online Users

Guests: 86
Record Number of Users: 8 on April 13, 2023
Record Number of Guests: 421 on December 02, 2018
Powered by nginx      Powered by FreeBSD      PHP Powered      Powered by MariaDB      ipv6 ready