# HG changeset patch
# User Roman Arutyunyan <arut@nginx.com>
# Date 1673867281 0
# Mon Jan 16 11:08:01 2023 +0000
# Branch quic
# Node ID 52681b76d4db0e9e814f1e69b0d6b35f5ac630a2
# Parent cc74c21cddd87762c03c5ce5a9976b5f23d8823f
QUIC: eBPF worker sockets.

For each nginx worker, a worker socket is created. Worker sockets process
existing QUIC connections bound to the worker, while listen sockets handle
new connections. When shutting down a worker, listen sockets are closed as
uaual, while worker sockets keep handling existing connections.

Reuseport eBPF program looks up a worker socket by packet DCID and, if not
found, chooses a listen socket based on packet address hash.

The mode is enabled by "quic_bpf on" directive and is only available on Linux.
Reuseport listen parameter is requried to enable the feature on a QUIC listen.

diff --git a/auto/modules b/auto/modules
--- a/auto/modules
+++ b/auto/modules
@@ -1363,11 +1363,11 @@ if [ $USE_OPENSSL_QUIC = YES ]; then

. auto/module

- if [ $QUIC_BPF = YES -a $SO_COOKIE_FOUND = YES ]; then
+ if [ $QUIC_BPF = YES ]; then
ngx_module_type=CORE
ngx_module_name=ngx_quic_bpf_module
ngx_module_incs=
- ngx_module_deps=
+ ngx_module_deps=src/event/quic/ngx_event_quic_bpf.h
ngx_module_srcs="src/event/quic/ngx_event_quic_bpf.c \
src/event/quic/ngx_event_quic_bpf_code.c"
ngx_module_libs=
diff --git a/auto/options b/auto/options
--- a/auto/options
+++ b/auto/options
@@ -171,8 +171,6 @@ USE_GEOIP=NO
NGX_GOOGLE_PERFTOOLS=NO
NGX_CPP_TEST=NO

-SO_COOKIE_FOUND=NO
-
NGX_LIBATOMIC=NO

NGX_CPU_CACHE_LINE=
diff --git a/auto/os/linux b/auto/os/linux
--- a/auto/os/linux
+++ b/auto/os/linux
@@ -234,7 +234,7 @@ ngx_include="sys/vfs.h"; . auto/incl

# BPF sockhash

-ngx_feature="BPF sockhash"
+ngx_feature="BPF maps"
ngx_feature_name="NGX_HAVE_BPF"
ngx_feature_run=no
ngx_feature_incs="#include <linux/bpf.h>
@@ -245,7 +245,14 @@ ngx_feature_test="union bpf_attr attr =

attr.map_flags = 0;
attr.map_type = BPF_MAP_TYPE_SOCKHASH;
+ syscall(__NR_bpf, 0, &attr, 0);

+ attr.map_flags = 0;
+ attr.map_type = BPF_MAP_TYPE_SOCKMAP;
+ syscall(__NR_bpf, 0, &attr, 0);
+
+ attr.map_flags = 0;
+ attr.map_type = BPF_MAP_TYPE_ARRAY;
syscall(__NR_bpf, 0, &attr, 0);"
. auto/feature

@@ -259,23 +266,6 @@ if [ $ngx_found = yes ]; then
fi


-ngx_feature="SO_COOKIE"
-ngx_feature_name="NGX_HAVE_SO_COOKIE"
-ngx_feature_run=no
-ngx_feature_incs="#include <sys/socket.h>
- #include <stdint.h>"
-ngx_feature_path=
-ngx_feature_libs=
-ngx_feature_test="socklen_t optlen = sizeof(uint64_t);
- uint64_t cookie;
- getsockopt(0, SOL_SOCKET, SO_COOKIE, &cookie, &optlen)"
-. auto/feature
-
-if [ $ngx_found = yes ]; then
- SO_COOKIE_FOUND=YES
-fi
-
-
# UDP segmentation offloading

ngx_feature="UDP_SEGMENT"
diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -1033,12 +1033,6 @@ ngx_close_listening_sockets(ngx_cycle_t
ls = cycle->listening.elts;
for (i = 0; i < cycle->listening.nelts; i++) {

-#if (NGX_QUIC)
- if (ls[i].quic) {
- continue;
- }
-#endif
-
c = ls[i].connection;

if (c) {
diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h
--- a/src/event/ngx_event.h
+++ b/src/event/ngx_event.h
@@ -73,6 +73,7 @@ struct ngx_event_s {
/* to test on worker exit */
unsigned channel:1;
unsigned resolver:1;
+ unsigned quic:1;

unsigned cancelable:1;

diff --git a/src/event/quic/bpf/makefile b/src/event/quic/bpf/makefile
--- a/src/event/quic/bpf/makefile
+++ b/src/event/quic/bpf/makefile
@@ -1,4 +1,4 @@
-CFLAGS=-O2 -Wall
+CFLAGS=-O2 -Wall $(MAKE_CFLAGS)

LICENSE=BSD

diff --git a/src/event/quic/bpf/ngx_quic_reuseport_helper.c b/src/event/quic/bpf/ngx_quic_reuseport_helper.c
--- a/src/event/quic/bpf/ngx_quic_reuseport_helper.c
+++ b/src/event/quic/bpf/ngx_quic_reuseport_helper.c
@@ -44,97 +44,93 @@ char _license[] SEC("license") = LICENSE
#define NGX_QUIC_SERVER_CID_LEN 20


-#define advance_data(nbytes) \
- offset += nbytes; \
- if (start + offset > end) { \
- debugmsg("cannot read %ld bytes at offset %ld", nbytes, offset); \
- goto failed; \
- } \
- data = start + offset - 1;
-
-
-#define ngx_quic_parse_uint64(p) \
- (((__u64)(p)[0] << 56) | \
- ((__u64)(p)[1] << 48) | \
- ((__u64)(p)[2] << 40) | \
- ((__u64)(p)[3] << 32) | \
- ((__u64)(p)[4] << 24) | \
- ((__u64)(p)[5] << 16) | \
- ((__u64)(p)[6] << 8) | \
- ((__u64)(p)[7]))
-
-/*
- * actual map object is created by the "bpf" system call,
- * all pointers to this variable are replaced by the bpf loader
- */
-struct bpf_map_def SEC("maps") ngx_quic_sockmap;
+struct bpf_map_def SEC("maps") ngx_quic_listen;
+struct bpf_map_def SEC("maps") ngx_quic_worker;
+struct bpf_map_def SEC("maps") ngx_quic_nlisten;


SEC(PROGNAME)
-int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx)
+int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) \
{
- int rc;
- __u64 key;
+ int rc, flags;
+ __u32 zero, *nsockets, ns;
size_t len, offset;
- unsigned char *start, *end, *data, *dcid;
+ unsigned char *start, *end, dcid[NGX_QUIC_SERVER_CID_LEN];

start = ctx->data;
- end = (unsigned char *) ctx->data_end;
- offset = 0;
+ end = ctx->data_end;

- advance_data(sizeof(struct udphdr)); /* data at UDP header */
- advance_data(1); /* data at QUIC flags */
-
- if (data[0] & NGX_QUIC_PKT_LONG) {
+ offset = sizeof(struct udphdr) + 1; /* UDP header + QUIC flags */
+ if (start + offset > end) {
+ goto bad_dgram;
+ }

- advance_data(4); /* data at QUIC version */
- advance_data(1); /* data at DCID len */
+ flags = start[offset - 1];
+ if (flags & NGX_QUIC_PKT_LONG) {

- len = data[0]; /* read DCID length */
-
- if (len < 8) {
- /* it's useless to search for key in such short DCID */
- return SK_PASS;
+ offset += 5; /* QUIC version + DCID len */
+ if (start + offset > end) {
+ goto bad_dgram;
}

- } else {
- len = NGX_QUIC_SERVER_CID_LEN;
+ len = start[offset - 1];
+ if (len != NGX_QUIC_SERVER_CID_LEN) {
+ goto new_conn;
+ }
+ }
+
+ if (start + offset + NGX_QUIC_SERVER_CID_LEN > end) {
+ goto bad_dgram;
+ }
+
+ memcpy(dcid, start + offset, NGX_QUIC_SERVER_CID_LEN);
+
+ rc = bpf_sk_select_reuseport(ctx, &ngx_quic_worker, dcid, 0);
+
+ if (rc == 0) {
+ debugmsg("nginx quic socket selected by dcid");
+ return SK_PASS;
}

- dcid = &data[1];
- advance_data(len); /* we expect the packet to have full DCID */
+ if (rc != -ENOENT) {
+ debugmsg("nginx quic bpf_sk_select_reuseport() failed:%d", rc);
+ return SK_DROP;
+ }
+
+new_conn:

- /* make verifier happy */
- if (dcid + sizeof(__u64) > end) {
- goto failed;
+ zero = 0;
+
+ nsockets = bpf_map_lookup_elem(&ngx_quic_nlisten, &zero);
+
+ if (nsockets == NULL) {
+ debugmsg("nginx quic nsockets undefined");
+ return SK_DROP;
}

- key = ngx_quic_parse_uint64(dcid);
-
- rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0);
+ ns = ctx->hash % *nsockets;

- switch (rc) {
- case 0:
- debugmsg("nginx quic socket selected by key 0x%llx", key);
- return SK_PASS;
+ rc = bpf_sk_select_reuseport(ctx, &ngx_quic_listen, &ns, 0);

- /* kernel returns positive error numbers, errno.h defines positive */
- case -ENOENT:
- debugmsg("nginx quic default route for key 0x%llx", key);
- /* let the default reuseport logic decide which socket to choose */
+ if (rc == 0) {
+ debugmsg("nginx quic socket selected by hash:%d", (int) ns);
return SK_PASS;
-
- default:
- debugmsg("nginx quic bpf_sk_select_reuseport err: %d key 0x%llx",
- rc, key);
- goto failed;
}

-failed:
- /*
- * SK_DROP will generate ICMP, but we may want to process "invalid" packet
- * in userspace quic to investigate further and finally react properly
- * (maybe ignore, maybe send something in response or close connection)
- */
- return SK_PASS;
+ if (rc != -ENOENT) {
+ debugmsg("nginx quic bpf_sk_select_reuseport() failed:%d", rc);
+ return SK_DROP;
+ }
+
+ (void) bpf_map_update_elem(&ngx_quic_nlisten, &zero, &ns, BPF_ANY);
+
+ debugmsg("nginx quic cut sockets array:%d", (int) ns);
+
+ return SK_DROP;
+
+bad_dgram:
+
+ debugmsg("nginx quic bad datagram");
+
+ return SK_DROP;
}
diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c
--- a/src/event/quic/ngx_event_quic.c
+++ b/src/event/quic/ngx_event_quic.c
@@ -304,6 +304,10 @@ ngx_quic_new_connection(ngx_connection_t
qc->congestion.ssthresh = (size_t) -1;
qc->congestion.recovery_start = ngx_current_msec;

+ if (c->fd == c->listening->fd) {
+ qc->listen_bound = 1;
+ }
+
if (pkt->validated && pkt->retried) {
qc->tp.retry_scid.len = pkt->dcid.len;
qc->tp.retry_scid.data = ngx_pstrdup(c->pool, &pkt->dcid);
@@ -413,6 +417,15 @@ ngx_quic_input_handler(ngx_event_t *rev)
return;
}

+ if (qc->listen_bound) {
+ c->fd = (ngx_socket_t) -1;
+
+ qc->error = NGX_QUIC_ERR_NO_ERROR;
+ qc->error_reason = "graceful shutdown";
+ ngx_quic_close_connection(c, NGX_ERROR);
+ return;
+ }
+
if (!qc->closing && qc->conf->shutdown) {
qc->conf->shutdown(c);
}
@@ -885,14 +898,6 @@ ngx_quic_handle_packet(ngx_connection_t
pkt->odcid = pkt->dcid;
}

- if (ngx_terminate || ngx_exiting) {
- if (conf->retry) {
- return ngx_quic_send_retry(c, conf, pkt);
- }
-
- return NGX_ERROR;
- }
-
c->log->action = "creating quic connection";

qc = ngx_quic_new_connection(c, conf, pkt);
diff --git a/src/event/quic/ngx_event_quic_bpf.c b/src/event/quic/ngx_event_quic_bpf.c
--- a/src/event/quic/ngx_event_quic_bpf.c
+++ b/src/event/quic/ngx_event_quic_bpf.c
@@ -6,6 +6,8 @@

#include <ngx_config.h>
#include <ngx_core.h>
+#include <ngx_event.h>
+#include <ngx_event_quic_connection.h>


#define NGX_QUIC_BPF_VARNAME "NGINX_BPF_MAPS"
@@ -26,39 +28,57 @@

typedef struct {
ngx_queue_t queue;
- int map_fd;
+
+ int listen_map;
+ int worker_map;
+ int nlisten_map;

struct sockaddr *sockaddr;
socklen_t socklen;
- ngx_uint_t unused; /* unsigned unused:1; */
-} ngx_quic_sock_group_t;
+
+ ngx_array_t listening;
+
+ ngx_uint_t nlisten;
+ ngx_uint_t old_nlisten;
+} ngx_quic_bpf_group_t;
+
+
+typedef struct {
+ ngx_socket_t fd;
+ ngx_listening_t *listening;
+ ngx_connection_t *connection;
+} ngx_quic_bpf_listening_t;


typedef struct {
ngx_flag_t enabled;
- ngx_uint_t map_size;
- ngx_queue_t groups; /* of ngx_quic_sock_group_t */
+ ngx_uint_t max_connection_ids;
+ ngx_uint_t max_workers;
+ ngx_queue_t groups;
} ngx_quic_bpf_conf_t;


static void *ngx_quic_bpf_create_conf(ngx_cycle_t *cycle);
+static char *ngx_quic_bpf_init_conf(ngx_cycle_t *cycle, void *conf);
static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle);

static void ngx_quic_bpf_cleanup(void *data);
static ngx_inline void ngx_quic_bpf_close(ngx_log_t *log, int fd,
const char *name);

-static ngx_quic_sock_group_t *ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf,
+static ngx_quic_bpf_group_t *ngx_quic_bpf_find_group(ngx_cycle_t *cycle,
+ ngx_listening_t *ls);
+static ngx_quic_bpf_group_t *ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle,
ngx_listening_t *ls);
-static ngx_quic_sock_group_t *ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle,
- struct sockaddr *sa, socklen_t socklen);
-static ngx_quic_sock_group_t *ngx_quic_bpf_create_group(ngx_cycle_t *cycle,
+static ngx_quic_bpf_group_t *ngx_quic_bpf_create_group(ngx_cycle_t *cycle,
ngx_listening_t *ls);
-static ngx_quic_sock_group_t *ngx_quic_bpf_get_group(ngx_cycle_t *cycle,
+static ngx_int_t ngx_quic_bpf_inherit_fd(ngx_cycle_t *cycle, int fd);
+static ngx_quic_bpf_group_t *ngx_quic_bpf_get_group(ngx_cycle_t *cycle,
ngx_listening_t *ls);
static ngx_int_t ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle,
ngx_listening_t *ls);
-static uint64_t ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log);
+static ngx_int_t ngx_quic_bpf_add_worker_socket(ngx_cycle_t *cycle,
+ ngx_quic_bpf_group_t *grp, ngx_listening_t *ls);

static ngx_int_t ngx_quic_bpf_export_maps(ngx_cycle_t *cycle);
static ngx_int_t ngx_quic_bpf_import_maps(ngx_cycle_t *cycle);
@@ -82,7 +102,7 @@ static ngx_command_t ngx_quic_bpf_comma
static ngx_core_module_t ngx_quic_bpf_module_ctx = {
ngx_string("quic_bpf"),
ngx_quic_bpf_create_conf,
- NULL
+ ngx_quic_bpf_init_conf
};


@@ -113,7 +133,6 @@ ngx_quic_bpf_create_conf(ngx_cycle_t *cy
}

bcf->enabled = NGX_CONF_UNSET;
- bcf->map_size = NGX_CONF_UNSET_UINT;

ngx_queue_init(&bcf->groups);

@@ -121,12 +140,41 @@ ngx_quic_bpf_create_conf(ngx_cycle_t *cy
}


+static char *
+ngx_quic_bpf_init_conf(ngx_cycle_t *cycle, void *conf)
+{
+ ngx_quic_bpf_conf_t *bcf = conf;
+
+ ngx_quic_bpf_conf_t *obcf;
+
+ ngx_conf_init_value(bcf->enabled, 0);
+
+ if (cycle->old_cycle->conf_ctx == NULL) {
+ return NGX_CONF_OK;
+ }
+
+ obcf = ngx_quic_bpf_get_conf(cycle->old_cycle);
+ if (obcf == NULL) {
+ return NGX_CONF_OK;
+ }
+
+ if (obcf->enabled != bcf->enabled) {
+ ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+ "cannot change \"quic_bpf\" after reload, ignoring");
+ bcf->enabled = obcf->enabled;
+ }
+
+ return NGX_CONF_OK;
+}
+
+
static ngx_int_t
ngx_quic_bpf_module_init(ngx_cycle_t *cycle)
{
ngx_uint_t i;
ngx_listening_t *ls;
ngx_core_conf_t *ccf;
+ ngx_event_conf_t *ecf;
ngx_pool_cleanup_t *cln;
ngx_quic_bpf_conf_t *bcf;

@@ -138,12 +186,16 @@ ngx_quic_bpf_module_init(ngx_cycle_t *cy
return NGX_OK;
}

- ccf = ngx_core_get_conf(cycle);
bcf = ngx_quic_bpf_get_conf(cycle);
+ if (!bcf->enabled) {
+ return NGX_OK;
+ }

- ngx_conf_init_value(bcf->enabled, 0);
+ ccf = ngx_core_get_conf(cycle);
+ ecf = ngx_event_get_conf(cycle->conf_ctx, ngx_event_core_module);

- bcf->map_size = ccf->worker_processes * 4;
+ bcf->max_connection_ids = ecf->connections * NGX_QUIC_MAX_SERVER_IDS;
+ bcf->max_workers = ccf->worker_processes * 4;

cln = ngx_pool_cleanup_add(cycle->pool, 0);
if (cln == NULL) {
@@ -153,6 +205,8 @@ ngx_quic_bpf_module_init(ngx_cycle_t *cy
cln->data = bcf;
cln->handler = ngx_quic_bpf_cleanup;

+ ls = cycle->listening.elts;
+
if (ngx_inherited && ngx_is_init_cycle(cycle->old_cycle)) {
if (ngx_quic_bpf_import_maps(cycle) != NGX_OK) {
goto failed;
@@ -208,16 +262,32 @@ ngx_quic_bpf_cleanup(void *data)
{
ngx_quic_bpf_conf_t *bcf = (ngx_quic_bpf_conf_t *) data;

- ngx_queue_t *q;
- ngx_quic_sock_group_t *grp;
+ ngx_uint_t i;
+ ngx_queue_t *q;
+ ngx_quic_bpf_group_t *grp;
+ ngx_quic_bpf_listening_t *bls;

for (q = ngx_queue_head(&bcf->groups);
q != ngx_queue_sentinel(&bcf->groups);
q = ngx_queue_next(q))
{
- grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);
+ grp = ngx_queue_data(q, ngx_quic_bpf_group_t, queue);
+
+ ngx_quic_bpf_close(ngx_cycle->log, grp->listen_map, "listen");
+ ngx_quic_bpf_close(ngx_cycle->log, grp->worker_map, "worker");
+ ngx_quic_bpf_close(ngx_cycle->log, grp->nlisten_map, "nlisten");
+
+ bls = grp->listening.elts;

- ngx_quic_bpf_close(ngx_cycle->log, grp->map_fd, "map");
+ for (i = 0; i < grp->listening.nelts; i++) {
+ if (bls[i].fd != (ngx_socket_t) -1) {
+ if (ngx_close_socket(bls[i].fd) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log,
+ ngx_socket_errno,
+ ngx_close_socket_n " failed");
+ }
+ }
+ }
}
}

@@ -230,25 +300,32 @@ ngx_quic_bpf_close(ngx_log_t *log, int f
}

ngx_log_error(NGX_LOG_EMERG, log, ngx_errno,
- "quic bpf close %s fd:%d failed", name, fd);
+ "QUIC BPF close %s map fd:%d failed", name, fd);
}


-static ngx_quic_sock_group_t *
-ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, ngx_listening_t *ls)
+static ngx_quic_bpf_group_t *
+ngx_quic_bpf_find_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
{
- ngx_queue_t *q;
- ngx_quic_sock_group_t *grp;
+ ngx_queue_t *q;
+ ngx_quic_bpf_conf_t *bcf;
+ ngx_quic_bpf_group_t *grp;
+
+ bcf = ngx_quic_bpf_get_conf(cycle);
+
+ if (!bcf->enabled || !ls->quic || !ls->reuseport) {
+ return NULL;
+ }

for (q = ngx_queue_head(&bcf->groups);
q != ngx_queue_sentinel(&bcf->groups);
q = ngx_queue_next(q))
{
- grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);
+ grp = ngx_queue_data(q, ngx_quic_bpf_group_t, queue);

if (ngx_cmp_sockaddr(ls->sockaddr, ls->socklen,
grp->sockaddr, grp->socklen, 1)
- == NGX_OK)
+ == 0)
{
return grp;
}
@@ -258,26 +335,32 @@ ngx_quic_bpf_find_group(ngx_quic_bpf_con
}


-static ngx_quic_sock_group_t *
-ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, struct sockaddr *sa,
- socklen_t socklen)
+static ngx_quic_bpf_group_t *
+ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
{
ngx_quic_bpf_conf_t *bcf;
- ngx_quic_sock_group_t *grp;
+ ngx_quic_bpf_group_t *grp;

bcf = ngx_quic_bpf_get_conf(cycle);

- grp = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_sock_group_t));
+ grp = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_bpf_group_t));
if (grp == NULL) {
return NULL;
}

- grp->socklen = socklen;
- grp->sockaddr = ngx_palloc(cycle->pool, socklen);
- if (grp->sockaddr == NULL) {
+ grp->listen_map = -1;
+ grp->worker_map = -1;
+ grp->nlisten_map = -1;
+
+ grp->sockaddr = ls->sockaddr;
+ grp->socklen = ls->socklen;
+
+ if (ngx_array_init(&grp->listening, cycle->pool, 1,
+ sizeof(ngx_quic_bpf_listening_t))
+ != NGX_OK)
+ {
return NULL;
}
- ngx_memcpy(grp->sockaddr, sa, socklen);

ngx_queue_insert_tail(&bcf->groups, &grp->queue);

@@ -285,50 +368,72 @@ ngx_quic_bpf_alloc_group(ngx_cycle_t *cy
}


-static ngx_quic_sock_group_t *
+static ngx_quic_bpf_group_t *
ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
{
- int progfd, failed, flags, rc;
- ngx_quic_bpf_conf_t *bcf;
- ngx_quic_sock_group_t *grp;
+ int progfd, failed;
+ ngx_quic_bpf_conf_t *bcf;
+ ngx_quic_bpf_group_t *grp;

bcf = ngx_quic_bpf_get_conf(cycle);

- if (!bcf->enabled) {
- return NULL;
- }
-
- grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen);
+ grp = ngx_quic_bpf_alloc_group(cycle, ls);
if (grp == NULL) {
return NULL;
}

- grp->map_fd = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKHASH,
- sizeof(uint64_t), sizeof(uint64_t),
- bcf->map_size, 0);
- if (grp->map_fd == -1) {
+ grp->listen_map = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKMAP,
+ sizeof(uint32_t), sizeof(uint64_t),
+ bcf->max_workers, 0);
+ if (grp->listen_map == -1) {
goto failed;
}

- flags = fcntl(grp->map_fd, F_GETFD);
- if (flags == -1) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, errno,
- "quic bpf getfd failed");
- goto failed;
- }
-
- /* need to inherit map during binary upgrade after exec */
- flags &= ~FD_CLOEXEC;
-
- rc = fcntl(grp->map_fd, F_SETFD, flags);
- if (rc == -1) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, errno,
- "quic bpf setfd failed");
+ if (ngx_quic_bpf_inherit_fd(cycle, grp->listen_map) != NGX_OK) {
goto failed;
}

ngx_bpf_program_link(&ngx_quic_reuseport_helper,
- "ngx_quic_sockmap", grp->map_fd);
+ "ngx_quic_listen", grp->listen_map);
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+ "quic bpf listen map created fd:%d", grp->listen_map);
+
+
+ grp->worker_map = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKHASH,
+ NGX_QUIC_SERVER_CID_LEN, sizeof(uint64_t),
+ bcf->max_connection_ids, 0);
+ if (grp->worker_map == -1) {
+ goto failed;
+ }
+
+ if (ngx_quic_bpf_inherit_fd(cycle, grp->worker_map) != NGX_OK) {
+ goto failed;
+ }
+
+ ngx_bpf_program_link(&ngx_quic_reuseport_helper,
+ "ngx_quic_worker", grp->worker_map);
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+ "quic bpf worker map created fd:%d", grp->worker_map);
+
+
+ grp->nlisten_map = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_ARRAY,
+ sizeof(uint32_t), sizeof(uint32_t), 1, 0);
+ if (grp->nlisten_map == -1) {
+ goto failed;
+ }
+
+ if (ngx_quic_bpf_inherit_fd(cycle, grp->nlisten_map) != NGX_OK) {
+ goto failed;
+ }
+
+ ngx_bpf_program_link(&ngx_quic_reuseport_helper,
+ "ngx_quic_nlisten", grp->nlisten_map);
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+ "quic bpf nlisten map created fd:%d", grp->nlisten_map);
+

progfd = ngx_bpf_load_program(cycle->log, &ngx_quic_reuseport_helper);
if (progfd < 0) {
@@ -352,14 +457,116 @@ ngx_quic_bpf_create_group(ngx_cycle_t *c
goto failed;
}

- ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
- "quic bpf sockmap created fd:%d", grp->map_fd);
return grp;

failed:

- if (grp->map_fd != -1) {
- ngx_quic_bpf_close(cycle->log, grp->map_fd, "map");
+ if (grp->listen_map != -1) {
+ ngx_quic_bpf_close(cycle->log, grp->listen_map, "listen");
+ }
+
+ if (grp->worker_map != -1) {
+ ngx_quic_bpf_close(cycle->log, grp->worker_map, "worker");
+ }
+
+ if (grp->nlisten_map != -1) {
+ ngx_quic_bpf_close(cycle->log, grp->nlisten_map, "nlisten");
+ }
+
+ ngx_queue_remove(&grp->queue);
+
+ return NULL;
+}
+
+
+static ngx_int_t
+ngx_quic_bpf_inherit_fd(ngx_cycle_t *cycle, int fd)
+{
+ int flags;
+
+ flags = fcntl(fd, F_GETFD);
+ if (flags == -1) {
+ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+ "fcntl(F_GETFD) failed");
+ return NGX_ERROR;
+ }
+
+ flags &= ~FD_CLOEXEC;
+
+ if (fcntl(fd, F_SETFD, flags) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+ "fcntl(F_SETFD) failed");
+ return NGX_ERROR;
+ }
+
+ return NGX_OK;
+}
+
+
+static ngx_quic_bpf_group_t *
+ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
+{
+ ngx_quic_bpf_conf_t *old_bcf;
+ ngx_quic_bpf_group_t *grp, *ogrp;
+
+ grp = ngx_quic_bpf_find_group(cycle, ls);
+ if (grp) {
+ return grp;
+ }
+
+ old_bcf = ngx_quic_bpf_get_old_conf(cycle);
+ if (old_bcf == NULL) {
+ return ngx_quic_bpf_create_group(cycle, ls);
+ }
+
+ ogrp = ngx_quic_bpf_find_group(cycle->old_cycle, ls);
+ if (ogrp == NULL) {
+ return ngx_quic_bpf_create_group(cycle, ls);
+ }
+
+ grp = ngx_quic_bpf_alloc_group(cycle, ls);
+ if (grp == NULL) {
+ return NULL;
+ }
+
+ grp->old_nlisten = ogrp->nlisten;
+
+ grp->listen_map = dup(ogrp->listen_map);
+ if (grp->listen_map == -1) {
+ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+ "failed to duplicate QUIC BPF listen map");
+
+ goto failed;
+ }
+
+ grp->worker_map = dup(ogrp->worker_map);
+ if (grp->worker_map == -1) {
+ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+ "failed to duplicate QUIC BPF worker map");
+ goto failed;
+ }
+
+ grp->nlisten_map = dup(ogrp->nlisten_map);
+ if (grp->nlisten_map == -1) {
+ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+ "failed to duplicate QUIC BPF nlisten map");
+ goto failed;
+ }
+
+ return grp;
+
+failed:
+
+ if (grp->listen_map != -1) {
+ ngx_quic_bpf_close(cycle->log, grp->listen_map, "listen");
+ }
+
+ if (grp->worker_map != -1) {
+ ngx_quic_bpf_close(cycle->log, grp->worker_map, "worker");
+ }
+
+ if (grp->nlisten_map != -1) {
+ ngx_quic_bpf_close(cycle->log, grp->nlisten_map, "nlisten");
}

ngx_queue_remove(&grp->queue);
@@ -368,129 +575,173 @@ failed:
}


-static ngx_quic_sock_group_t *
-ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
-{
- ngx_quic_bpf_conf_t *bcf, *old_bcf;
- ngx_quic_sock_group_t *grp, *ogrp;
-
- bcf = ngx_quic_bpf_get_conf(cycle);
-
- grp = ngx_quic_bpf_find_group(bcf, ls);
- if (grp) {
- return grp;
- }
-
- old_bcf = ngx_quic_bpf_get_old_conf(cycle);
-
- if (old_bcf == NULL) {
- return ngx_quic_bpf_create_group(cycle, ls);
- }
-
- ogrp = ngx_quic_bpf_find_group(old_bcf, ls);
- if (ogrp == NULL) {
- return ngx_quic_bpf_create_group(cycle, ls);
- }
-
- grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen);
- if (grp == NULL) {
- return NULL;
- }
-
- grp->map_fd = dup(ogrp->map_fd);
- if (grp->map_fd == -1) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
- "quic bpf failed to duplicate bpf map descriptor");
-
- ngx_queue_remove(&grp->queue);
-
- return NULL;
- }
-
- ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
- "quic bpf sockmap fd duplicated old:%d new:%d",
- ogrp->map_fd, grp->map_fd);
-
- return grp;
-}
-
-
static ngx_int_t
ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle, ngx_listening_t *ls)
{
- uint64_t cookie;
- ngx_quic_bpf_conf_t *bcf;
- ngx_quic_sock_group_t *grp;
-
- bcf = ngx_quic_bpf_get_conf(cycle);
+ uint32_t zero, key;
+ ngx_quic_bpf_group_t *grp;

grp = ngx_quic_bpf_get_group(cycle, ls);
+ if (grp == NULL) {
+ return NGX_ERROR;
+ }

- if (grp == NULL) {
- if (!bcf->enabled) {
- return NGX_OK;
- }
-
+ if (ngx_quic_bpf_add_worker_socket(cycle, grp, ls) != NGX_OK) {
return NGX_ERROR;
}

- grp->unused = 0;
+ key = ls->worker;

- cookie = ngx_quic_bpf_socket_key(ls->fd, cycle->log);
- if (cookie == (uint64_t) NGX_ERROR) {
+ if (ngx_bpf_map_update(grp->listen_map, &key, &ls->fd, BPF_ANY) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+ "failed to update QUIC BPF listen map");
return NGX_ERROR;
}

- /* map[cookie] = socket; for use in kernel helper */
- if (ngx_bpf_map_update(grp->map_fd, &cookie, &ls->fd, BPF_ANY) == -1) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
- "quic bpf failed to update socket map key=%xL", cookie);
- return NGX_ERROR;
+ if (grp->nlisten >= ls->worker + 1) {
+ return NGX_OK;
}

- ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
- "quic bpf sockmap fd:%d add socket:%d cookie:0x%xL worker:%ui",
- grp->map_fd, ls->fd, cookie, ls->worker);
+ grp->nlisten = ls->worker + 1;
+
+ if (grp->nlisten <= grp->old_nlisten) {
+ return NGX_OK;
+ }

- /* do not inherit this socket */
- ls->ignore = 1;
+ zero = 0;
+ key = grp->nlisten;
+
+ if (ngx_bpf_map_update(grp->nlisten_map, &zero, &key, BPF_ANY) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+ "failed to update QUIC BPF nlisten map");
+ return NGX_ERROR;
+ }

return NGX_OK;
}


-static uint64_t
-ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log)
+static ngx_int_t
+ngx_quic_bpf_add_worker_socket(ngx_cycle_t *cycle, ngx_quic_bpf_group_t *grp,
+ ngx_listening_t *ls)
{
- uint64_t cookie;
- socklen_t optlen;
+ int value;
+ ngx_uint_t i, n;
+ ngx_socket_t s;
+ ngx_quic_bpf_listening_t *bls;
+
+ s = ngx_socket(ls->sockaddr->sa_family, SOCK_DGRAM, 0);
+ if (s == (ngx_socket_t) -1) {
+ ngx_log_error(NGX_LOG_ERR, cycle->log, ngx_socket_errno,
+ ngx_socket_n " failed");
+ return NGX_ERROR;
+ }

- optlen = sizeof(cookie);
+ if (ngx_nonblocking(s) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
+ ngx_nonblocking_n " worker socket failed");
+ goto failed;
+ }
+
+ value = 1;

- if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) {
- ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
- "quic bpf getsockopt(SO_COOKIE) failed");
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR,
+ (const void *) &value, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
+ "setsockopt(SO_REUSEADDR) worker socket failed");
+ goto failed;
+ }

- return (ngx_uint_t) NGX_ERROR;
+ if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+ (const void *) &value, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
+ "setsockopt(SO_REUSEPORT) worker socket failed");
+ goto failed;
}

- return cookie;
+#if (NGX_HAVE_IP_PKTINFO)
+ if (ls->wildcard && ls->sockaddr->sa_family == AF_INET) {
+ if (setsockopt(s, IPPROTO_IP, IP_PKTINFO,
+ (const void *) &value, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
+ "setsockopt(IP_PKTINFO) worker socket failed");
+ goto failed;
+ }
+ }
+#endif
+
+#if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO)
+ if (ls->wildcard && ls->sockaddr->sa_family == AF_INET6) {
+ if (setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO,
+ (const void *) &value, sizeof(int))
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
+ "setsockopt(IPV6_RECVPKTINFO) worker socket failed");
+ }
+ }
+#endif
+
+ if (bind(s, ls->sockaddr, ls->socklen) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
+ "bind() failed");
+ goto failed;
+ }
+
+ if (ls->worker >= grp->listening.nelts) {
+ n = ls->worker + 1 - grp->listening.nelts;
+
+ bls = ngx_array_push_n(&grp->listening, n);
+ if (bls == NULL) {
+ goto failed;
+ }
+
+ ngx_memzero(bls, n * sizeof(ngx_quic_bpf_listening_t));
+
+ for (i = 0; i < n; i++) {
+ bls[i].fd = (ngx_socket_t) -1;
+ }
+ }
+
+ bls = grp->listening.elts;
+ bls[ls->worker].fd = s;
+ bls[ls->worker].listening = ls;
+
+ return NGX_OK;
+
+failed:
+
+ if (ngx_close_socket(s) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno,
+ ngx_close_socket_n " failed");
+ }
+
+ return NGX_ERROR;
}

-
static ngx_int_t
ngx_quic_bpf_export_maps(ngx_cycle_t *cycle)
{
- u_char *p, *buf;
- size_t len;
- ngx_str_t *var;
- ngx_queue_t *q;
- ngx_core_conf_t *ccf;
- ngx_quic_bpf_conf_t *bcf;
- ngx_quic_sock_group_t *grp;
+ u_char *p, *buf;
+ size_t len;
+ ngx_str_t *var;
+ ngx_queue_t *q;
+ ngx_core_conf_t *ccf;
+ ngx_quic_bpf_conf_t *bcf;
+ ngx_quic_bpf_group_t *grp;
+
+ bcf = ngx_quic_bpf_get_conf(cycle);
+ if (!bcf->enabled) {
+ return NGX_OK;
+ }

ccf = ngx_core_get_conf(cycle);
- bcf = ngx_quic_bpf_get_conf(cycle);

len = sizeof(NGX_QUIC_BPF_VARNAME) + 1;

@@ -498,24 +749,26 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cy

while (q != ngx_queue_sentinel(&bcf->groups)) {

- grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);
+ grp = ngx_queue_data(q, ngx_quic_bpf_group_t, queue);

q = ngx_queue_next(q);

- if (grp->unused) {
+ if (grp->nlisten == 0) {
/*
* map was inherited, but it is not used in this configuration;
* do not pass such map further and drop the group to prevent
* interference with changes during reload
*/

- ngx_quic_bpf_close(cycle->log, grp->map_fd, "map");
+ ngx_quic_bpf_close(cycle->log, grp->listen_map, "listen");
+ ngx_quic_bpf_close(cycle->log, grp->worker_map, "worker");
+ ngx_quic_bpf_close(cycle->log, grp->nlisten_map, "nlisten");
+
ngx_queue_remove(&grp->queue);
-
continue;
}

- len += NGX_INT32_LEN + 1 + NGX_SOCKADDR_STRLEN + 1;
+ len += (NGX_INT32_LEN + 1) * 3 + NGX_SOCKADDR_STRLEN + 1;
}

len++;
@@ -525,22 +778,23 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cy
return NGX_ERROR;
}

- p = ngx_cpymem(buf, NGX_QUIC_BPF_VARNAME "=",
- sizeof(NGX_QUIC_BPF_VARNAME));
+ p = ngx_cpymem(buf, NGX_QUIC_BPF_VARNAME "=", sizeof(NGX_QUIC_BPF_VARNAME));

for (q = ngx_queue_head(&bcf->groups);
q != ngx_queue_sentinel(&bcf->groups);
q = ngx_queue_next(q))
{
- grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue);
+ grp = ngx_queue_data(q, ngx_quic_bpf_group_t, queue);

- p = ngx_sprintf(p, "%ud", grp->map_fd);
-
+ p = ngx_sprintf(p, "%ud", grp->listen_map);
+ *p++ = NGX_QUIC_BPF_ADDRSEP;
+ p = ngx_sprintf(p, "%ud", grp->worker_map);
+ *p++ = NGX_QUIC_BPF_ADDRSEP;
+ p = ngx_sprintf(p, "%ud", grp->nlisten_map);
*p++ = NGX_QUIC_BPF_ADDRSEP;

p += ngx_sock_ntop(grp->sockaddr, grp->socklen, p,
NGX_SOCKADDR_STRLEN, 1);
-
*p++ = NGX_QUIC_BPF_VARSEP;
}

@@ -561,12 +815,14 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cy
static ngx_int_t
ngx_quic_bpf_import_maps(ngx_cycle_t *cycle)
{
- int s;
- u_char *inherited, *p, *v;
- ngx_uint_t in_fd;
- ngx_addr_t tmp;
- ngx_quic_bpf_conf_t *bcf;
- ngx_quic_sock_group_t *grp;
+ int fds[3];
+ u_char *inherited, *p, *v;
+ uint32_t zero, nlisten;
+ ngx_int_t fd;
+ ngx_uint_t nfd;
+ ngx_addr_t tmp;
+ ngx_quic_bpf_conf_t *bcf;
+ ngx_quic_bpf_group_t *grp;

inherited = (u_char *) getenv(NGX_QUIC_BPF_VARNAME);

@@ -574,13 +830,13 @@ ngx_quic_bpf_import_maps(ngx_cycle_t *cy
return NGX_OK;
}

+ ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0,
+ "using inherited QUIC BPF maps from \"%s\"", inherited);
+
bcf = ngx_quic_bpf_get_conf(cycle);

-#if (NGX_SUPPRESS_WARN)
- s = -1;
-#endif
-
- in_fd = 1;
+ zero = 0;
+ nfd = 0;

for (p = inherited, v = p; *p; p++) {

@@ -588,63 +844,69 @@ ngx_quic_bpf_import_maps(ngx_cycle_t *cy

case NGX_QUIC_BPF_ADDRSEP:

- if (!in_fd) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
- "quic bpf failed to parse inherited env");
- return NGX_ERROR;
- }
- in_fd = 0;
-
- s = ngx_atoi(v, p - v);
- if (s == NGX_ERROR) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
- "quic bpf failed to parse inherited map fd");
- return NGX_ERROR;
+ if (nfd > 2) {
+ goto failed;
}

+ fd = ngx_atoi(v, p - v);
+ if (fd == NGX_ERROR) {
+ goto failed;
+ }
+
+ fds[nfd++] = fd;
v = p + 1;
break;

case NGX_QUIC_BPF_VARSEP:

- if (in_fd) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
- "quic bpf failed to parse inherited env");
- return NGX_ERROR;
+ if (nfd != 3) {
+ goto failed;
}
- in_fd = 1;

- grp = ngx_pcalloc(cycle->pool,
- sizeof(ngx_quic_sock_group_t));
+ grp = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_bpf_group_t));
if (grp == NULL) {
return NGX_ERROR;
}

- grp->map_fd = s;
-
- if (ngx_parse_addr_port(cycle->pool, &tmp, v, p - v)
+ if (ngx_array_init(&grp->listening, cycle->pool, 1,
+ sizeof(ngx_quic_bpf_listening_t))
!= NGX_OK)
{
- ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
- "quic bpf failed to parse inherited"
- " address '%*s'", p - v , v);
+ return NGX_ERROR;
+ }
+
+ grp->listen_map = fds[0];
+ grp->worker_map = fds[1];
+ grp->nlisten_map = fds[2];

- ngx_quic_bpf_close(cycle->log, s, "inherited map");
+ if (ngx_parse_addr_port(cycle->pool, &tmp, v, p - v) != NGX_OK) {
+ goto failed;
+ }

+ grp->sockaddr = ngx_pcalloc(cycle->pool, tmp.socklen);
+ if (grp->sockaddr == NULL) {
return NGX_ERROR;
}

- grp->sockaddr = tmp.sockaddr;
+ ngx_memcpy(grp->sockaddr, tmp.sockaddr, tmp.socklen);
grp->socklen = tmp.socklen;

- grp->unused = 1;
+ if (ngx_bpf_map_lookup(fds[2], &zero, &nlisten) == -1) {
+ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+ "failed to lookup QUIC BPF listen number");
+ return NGX_ERROR;
+ }
+
+ grp->old_nlisten = nlisten;

ngx_queue_insert_tail(&bcf->groups, &grp->queue);

- ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+ ngx_log_debug5(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"quic bpf sockmap inherited with "
- "fd:%d address:%*s",
- grp->map_fd, p - v, v);
+ "fds:%d/%d/%d address:%*s",
+ fds[0], fds[1], fds[2], p - v, v);
+
+ nfd = 0;
v = p + 1;
break;

@@ -654,4 +916,127 @@ ngx_quic_bpf_import_maps(ngx_cycle_t *cy
}

return NGX_OK;
+
+failed:
+
+ ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+ "failed to parse inherited QUIC BPF variable");
+
+ return NGX_ERROR;
}
+
+
+ngx_int_t
+ngx_quic_bpf_get_client_connection(ngx_connection_t *lc, ngx_connection_t **pc)
+{
+ ngx_event_t *rev;
+ ngx_connection_t *c;
+ ngx_quic_bpf_group_t *grp;
+ ngx_quic_bpf_listening_t *bpf_listening, *bls;
+
+ grp = ngx_quic_bpf_find_group((ngx_cycle_t *) ngx_cycle, lc->listening);
+
+ if (grp == NULL || ngx_worker >= grp->listening.nelts) {
+ return NGX_OK;
+ }
+
+ bpf_listening = grp->listening.elts;
+ bls = &bpf_listening[ngx_worker];
+
+ if (bls->fd == (ngx_socket_t) -1) {
+ return NGX_OK;
+ }
+
+ if (bls->connection == NULL) {
+ c = ngx_get_connection(bls->fd, lc->log);
+ if (c == NULL) {
+ return NGX_ERROR;
+ }
+
+ c->type = SOCK_DGRAM;
+ c->log = lc->log;
+ c->listening = bls->listening;
+
+ rev = c->read;
+ rev->quic = 1;
+ rev->log = c->log;
+ rev->handler = ngx_quic_recvmsg;
+
+ if (ngx_add_event(rev, NGX_READ_EVENT, 0) == NGX_ERROR) {
+ ngx_free_connection(c);
+ return NGX_ERROR;
+ }
+
+ bls->connection = c;
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, lc->log, 0,
+ "quic bpf worker socket connection fd:%d", bls->fd);
+
+ }
+
+ *pc = ngx_get_connection(bls->fd, lc->log);
+ if (*pc == NULL) {
+ return NGX_ERROR;
+ }
+
+ (*pc)->shared = 1;
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, lc->log, 0,
+ "quic bpf client connection fd:%d", bls->fd);
+
+ return NGX_OK;
+}
+
+
+ngx_int_t
+ngx_quic_bpf_insert(ngx_connection_t *c, ngx_quic_connection_t *qc,
+ ngx_quic_socket_t *qsock)
+{
+ ngx_quic_bpf_group_t *grp;
+
+ if (qsock->sid.len != NGX_QUIC_SERVER_CID_LEN) {
+ /* route by address */
+ return NGX_OK;
+ }
+
+ grp = ngx_quic_bpf_find_group((ngx_cycle_t *) ngx_cycle, c->listening);
+ if (grp == NULL) {
+ return NGX_OK;
+ }
+
+ if (ngx_bpf_map_update(grp->worker_map, qsock->sid.id, &c->fd, BPF_ANY)
+ == -1)
+ {
+ ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+ "failed to update QUIC BPF worker map");
+ return NGX_ERROR;
+ }
+
+ return NGX_OK;
+}
+
+
+ngx_int_t
+ngx_quic_bpf_delete(ngx_connection_t *c, ngx_quic_connection_t *qc,
+ ngx_quic_socket_t *qsock)
+{
+ ngx_quic_bpf_group_t *grp;
+
+ if (qsock->sid.len != NGX_QUIC_SERVER_CID_LEN) {
+ /* route by address */
+ return NGX_OK;
+ }
+
+ grp = ngx_quic_bpf_find_group((ngx_cycle_t *) ngx_cycle, c->listening);
+ if (grp == NULL) {
+ return NGX_OK;
+ }
+
+ if (ngx_bpf_map_delete(grp->worker_map, qsock->sid.id) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+ "failed to update QUIC BPF worker map");
+ return NGX_ERROR;
+ }
+
+ return NGX_OK;
+}
diff --git a/src/event/quic/ngx_event_quic_bpf.h b/src/event/quic/ngx_event_quic_bpf.h
new file mode 100644
--- /dev/null
+++ b/src/event/quic/ngx_event_quic_bpf.h
@@ -0,0 +1,23 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_EVENT_QUIC_BPF_H_INCLUDED_
+#define _NGX_EVENT_QUIC_BPF_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+
+ngx_int_t ngx_quic_bpf_get_client_connection(ngx_connection_t *lc,
+ ngx_connection_t **pc);
+ngx_int_t ngx_quic_bpf_insert(ngx_connection_t *c, ngx_quic_connection_t *qc,
+ ngx_quic_socket_t *qsock);
+ngx_int_t ngx_quic_bpf_delete(ngx_connection_t *c, ngx_quic_connection_t *qc,
+ ngx_quic_socket_t *qsock);
+
+
+#endif /* _NGX_EVENT_QUIC_BPF_H_INCLUDED_ */
diff --git a/src/event/quic/ngx_event_quic_bpf_code.c b/src/event/quic/ngx_event_quic_bpf_code.c
--- a/src/event/quic/ngx_event_quic_bpf_code.c
+++ b/src/event/quic/ngx_event_quic_bpf_code.c
@@ -7,71 +7,146 @@


static ngx_bpf_reloc_t bpf_reloc_prog_ngx_quic_reuseport_helper[] = {
- { "ngx_quic_sockmap", 55 },
+ { "ngx_quic_worker", 82 },
+ { "ngx_quic_nlisten", 99 },
+ { "ngx_quic_listen", 110 },
+ { "ngx_quic_nlisten", 127 },
};

static struct bpf_insn bpf_insn_prog_ngx_quic_reuseport_helper[] = {
/* opcode dst src offset imm */
- { 0x79, BPF_REG_4, BPF_REG_1, (int16_t) 0, 0x0 },
- { 0x79, BPF_REG_3, BPF_REG_1, (int16_t) 8, 0x0 },
- { 0xbf, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 },
- { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 },
- { 0x2d, BPF_REG_2, BPF_REG_3, (int16_t) 54, 0x0 },
- { 0xbf, BPF_REG_5, BPF_REG_4, (int16_t) 0, 0x0 },
- { 0x7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x9 },
- { 0x2d, BPF_REG_5, BPF_REG_3, (int16_t) 51, 0x0 },
- { 0xb7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x14 },
- { 0xb7, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x9 },
- { 0x71, BPF_REG_6, BPF_REG_2, (int16_t) 0, 0x0 },
- { 0x67, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x38 },
- { 0xc7, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x38 },
- { 0x65, BPF_REG_6, BPF_REG_0, (int16_t) 10, 0xffffffff },
- { 0xbf, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 },
- { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xd },
- { 0x2d, BPF_REG_2, BPF_REG_3, (int16_t) 42, 0x0 },
- { 0xbf, BPF_REG_5, BPF_REG_4, (int16_t) 0, 0x0 },
- { 0x7, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0xe },
- { 0x2d, BPF_REG_5, BPF_REG_3, (int16_t) 39, 0x0 },
- { 0xb7, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xe },
- { 0x71, BPF_REG_5, BPF_REG_2, (int16_t) 0, 0x0 },
- { 0xb7, BPF_REG_6, BPF_REG_0, (int16_t) 0, 0x8 },
- { 0x2d, BPF_REG_6, BPF_REG_5, (int16_t) 35, 0x0 },
- { 0xf, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x0 },
- { 0xf, BPF_REG_4, BPF_REG_5, (int16_t) 0, 0x0 },
- { 0x2d, BPF_REG_4, BPF_REG_3, (int16_t) 32, 0x0 },
- { 0xbf, BPF_REG_4, BPF_REG_2, (int16_t) 0, 0x0 },
- { 0x7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x9 },
- { 0x2d, BPF_REG_4, BPF_REG_3, (int16_t) 29, 0x0 },
- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 1, 0x0 },
+ { 0xbf, BPF_REG_6, BPF_REG_1, (int16_t) 0, 0x0 },
+ { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x79, BPF_REG_2, BPF_REG_6, (int16_t) 8, 0x0 },
+ { 0x79, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 },
+ { 0xbf, BPF_REG_3, BPF_REG_1, (int16_t) 0, 0x0 },
+ { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x9 },
+ { 0x2d, BPF_REG_3, BPF_REG_2, (int16_t) 124, 0x0 },
+ { 0xb7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x9 },
+ { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 8, 0x0 },
{ 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x38 },
- { 0x71, BPF_REG_3, BPF_REG_2, (int16_t) 2, 0x0 },
- { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x30 },
- { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 },
- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 3, 0x0 },
- { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x28 },
- { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 },
- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 4, 0x0 },
- { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x20 },
- { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 },
- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 5, 0x0 },
- { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x18 },
+ { 0xc7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x38 },
+ { 0x65, BPF_REG_4, BPF_REG_0, (int16_t) 6, 0xffffffff },
+ { 0xbf, BPF_REG_3, BPF_REG_1, (int16_t) 0, 0x0 },
+ { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xe },
+ { 0x2d, BPF_REG_3, BPF_REG_2, (int16_t) 116, 0x0 },
+ { 0xb7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xe },
+ { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 13, 0x0 },
+ { 0x55, BPF_REG_4, BPF_REG_0, (int16_t) 77, 0x14 },
+ { 0xf, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 },
+ { 0xbf, BPF_REG_3, BPF_REG_1, (int16_t) 0, 0x0 },
+ { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x14 },
+ { 0x2d, BPF_REG_3, BPF_REG_2, (int16_t) 109, 0x0 },
+ { 0x71, BPF_REG_3, BPF_REG_1, (int16_t) 13, 0x0 },
+ { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x8 },
+ { 0x71, BPF_REG_2, BPF_REG_1, (int16_t) 12, 0x0 },
+ { 0x4f, BPF_REG_3, BPF_REG_2, (int16_t) 0, 0x0 },
+ { 0x71, BPF_REG_2, BPF_REG_1, (int16_t) 15, 0x0 },
+ { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 },
+ { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 14, 0x0 },
+ { 0x4f, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 },
+ { 0x71, BPF_REG_5, BPF_REG_1, (int16_t) 9, 0x0 },
+ { 0x67, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x8 },
+ { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 8, 0x0 },
+ { 0x4f, BPF_REG_5, BPF_REG_4, (int16_t) 0, 0x0 },
+ { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 11, 0x0 },
+ { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x8 },
+ { 0x71, BPF_REG_0, BPF_REG_1, (int16_t) 10, 0x0 },
+ { 0x4f, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x10 },
+ { 0x4f, BPF_REG_4, BPF_REG_5, (int16_t) 0, 0x0 },
+ { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x10 },
+ { 0x4f, BPF_REG_2, BPF_REG_3, (int16_t) 0, 0x0 },
+ { 0x71, BPF_REG_3, BPF_REG_1, (int16_t) 17, 0x0 },
+ { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x8 },
+ { 0x71, BPF_REG_5, BPF_REG_1, (int16_t) 16, 0x0 },
+ { 0x4f, BPF_REG_3, BPF_REG_5, (int16_t) 0, 0x0 },
+ { 0x71, BPF_REG_5, BPF_REG_1, (int16_t) 19, 0x0 },
+ { 0x67, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x8 },
+ { 0x71, BPF_REG_0, BPF_REG_1, (int16_t) 18, 0x0 },
+ { 0x4f, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 },
+ { 0x4f, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 },
+ { 0x67, BPF_REG_5, BPF_REG_0, (int16_t) 0, 0x10 },
+ { 0x4f, BPF_REG_5, BPF_REG_3, (int16_t) 0, 0x0 },
+ { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 1, 0x0 },
+ { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x8 },
+ { 0x71, BPF_REG_3, BPF_REG_1, (int16_t) 0, 0x0 },
+ { 0x4f, BPF_REG_4, BPF_REG_3, (int16_t) 0, 0x0 },
+ { 0x71, BPF_REG_3, BPF_REG_1, (int16_t) 3, 0x0 },
+ { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x8 },
+ { 0x71, BPF_REG_0, BPF_REG_1, (int16_t) 2, 0x0 },
+ { 0x4f, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x63, BPF_REG_10, BPF_REG_5, (int16_t) 65520, 0x0 },
+ { 0x7b, BPF_REG_10, BPF_REG_2, (int16_t) 65512, 0x0 },
+ { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x10 },
{ 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 },
- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 6, 0x0 },
- { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x10 },
- { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 },
- { 0x71, BPF_REG_4, BPF_REG_2, (int16_t) 7, 0x0 },
- { 0x67, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x8 },
- { 0x4f, BPF_REG_3, BPF_REG_4, (int16_t) 0, 0x0 },
- { 0x71, BPF_REG_2, BPF_REG_2, (int16_t) 8, 0x0 },
- { 0x4f, BPF_REG_3, BPF_REG_2, (int16_t) 0, 0x0 },
- { 0x7b, BPF_REG_10, BPF_REG_3, (int16_t) 65528, 0x0 },
+ { 0x71, BPF_REG_2, BPF_REG_1, (int16_t) 5, 0x0 },
+ { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 },
+ { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 4, 0x0 },
+ { 0x4f, BPF_REG_2, BPF_REG_4, (int16_t) 0, 0x0 },
+ { 0x71, BPF_REG_4, BPF_REG_1, (int16_t) 6, 0x0 },
+ { 0x71, BPF_REG_1, BPF_REG_1, (int16_t) 7, 0x0 },
+ { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x8 },
+ { 0x4f, BPF_REG_1, BPF_REG_4, (int16_t) 0, 0x0 },
+ { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x10 },
+ { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 },
+ { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x20 },
+ { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 },
+ { 0x7b, BPF_REG_10, BPF_REG_1, (int16_t) 65504, 0x0 },
{ 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 },
- { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff8 },
+ { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xffffffe0 },
+ { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 },
{ 0x18, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x0 },
{ 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 },
{ 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 },
{ 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x52 },
- { 0xb7, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1 },
+ { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x1 },
+ { 0x67, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 },
+ { 0x77, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 },
+ { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 41, 0x0 },
+ { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xfffffffe },
+ { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x1d, BPF_REG_0, BPF_REG_1, (int16_t) 2, 0x0 },
+ { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 36, 0x0 },
+ { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x63, BPF_REG_10, BPF_REG_7, (int16_t) 65532, 0x0 },
+ { 0xbf, BPF_REG_2, BPF_REG_10, (int16_t) 0, 0x0 },
+ { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xfffffffc },
+ { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1 },
+ { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 28, 0x0 },
+ { 0x61, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x61, BPF_REG_2, BPF_REG_6, (int16_t) 32, 0x0 },
+ { 0x9f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 },
+ { 0x63, BPF_REG_10, BPF_REG_2, (int16_t) 65528, 0x0 },
+ { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 },
+ { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff8 },
+ { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 },
+ { 0x18, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x52 },
+ { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x1 },
+ { 0x67, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 },
+ { 0x77, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 },
+ { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 13, 0x0 },
+ { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xfffffffe },
+ { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x1d, BPF_REG_0, BPF_REG_1, (int16_t) 1, 0x0 },
+ { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 65507, 0x0 },
+ { 0xbf, BPF_REG_2, BPF_REG_10, (int16_t) 0, 0x0 },
+ { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xfffffffc },
+ { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 },
+ { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff8 },
+ { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 },
+ { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x2 },
+ { 0xbf, BPF_REG_0, BPF_REG_7, (int16_t) 0, 0x0 },
{ 0x95, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 },
};

@@ -86,3 +161,4 @@ ngx_bpf_program_t ngx_quic_reuseport_hel
.license = "BSD",
.type = BPF_PROG_TYPE_SK_REUSEPORT,
};
+
diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h
--- a/src/event/quic/ngx_event_quic_connection.h
+++ b/src/event/quic/ngx_event_quic_connection.h
@@ -36,6 +36,9 @@ typedef struct ngx_quic_keys_s ng
#include <ngx_event_quic_ack.h>
#include <ngx_event_quic_output.h>
#include <ngx_event_quic_socket.h>
+#if (NGX_QUIC_BPF)
+#include <ngx_event_quic_bpf.h>
+#endif


/* RFC 9002, 6.2.2. Handshakes and New Paths: kInitialRtt */
@@ -45,6 +48,8 @@ typedef struct ngx_quic_keys_s ng

#define NGX_QUIC_SEND_CTX_LAST (NGX_QUIC_ENCRYPTION_LAST - 1)

+#define NGX_QUIC_MAX_SERVER_IDS 8
+
/* 0-RTT and 1-RTT data exist in the same packet number space,
* so we have 3 packet number spaces:
*
@@ -257,6 +262,7 @@ struct ngx_quic_connection_s {
unsigned key_phase:1;
unsigned validated:1;
unsigned client_tp_done:1;
+ unsigned listen_bound:1;
};


diff --git a/src/event/quic/ngx_event_quic_connid.c b/src/event/quic/ngx_event_quic_connid.c
--- a/src/event/quic/ngx_event_quic_connid.c
+++ b/src/event/quic/ngx_event_quic_connid.c
@@ -9,12 +9,7 @@
#include <ngx_event.h>
#include <ngx_event_quic_connection.h>

-#define NGX_QUIC_MAX_SERVER_IDS 8

-
-#if (NGX_QUIC_BPF)
-static ngx_int_t ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id);
-#endif
static ngx_int_t ngx_quic_retire_client_id(ngx_connection_t *c,
ngx_quic_client_id_t *cid);
static ngx_quic_client_id_t *ngx_quic_alloc_client_id(ngx_connection_t *c,
@@ -30,46 +25,10 @@ ngx_quic_create_server_id(ngx_connection
return NGX_ERROR;
}

-#if (NGX_QUIC_BPF)
- if (ngx_quic_bpf_attach_id(c, id) != NGX_OK) {
- ngx_log_error(NGX_LOG_ERR, c->log, 0,
- "quic bpf failed to generate socket key");
- /* ignore error, things still may work */
- }
-#endif
-
return NGX_OK;
}


-#if (NGX_QUIC_BPF)
-
-static ngx_int_t
-ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id)
-{
- int fd;
- uint64_t cookie;
- socklen_t optlen;
-
- fd = c->listening->fd;
-
- optlen = sizeof(cookie);
-
- if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) {
- ngx_log_error(NGX_LOG_ERR, c->log, ngx_socket_errno,
- "quic getsockopt(SO_COOKIE) failed");
-
- return NGX_ERROR;
- }
-
- ngx_quic_dcid_encode_key(id, cookie);
-
- return NGX_OK;
-}
-
-#endif
-
-
ngx_int_t
ngx_quic_handle_new_connection_id_frame(ngx_connection_t *c,
ngx_quic_new_conn_id_frame_t *f)
diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -84,6 +84,10 @@ ngx_quic_output(ngx_connection_t *c)
ngx_quic_congestion_t *cg;
ngx_quic_connection_t *qc;

+ if (c->fd == (ngx_socket_t) -1) {
+ return NGX_ERROR;
+ }
+
c->log->action = "sending frames";

qc = ngx_quic_get_connection(c);
@@ -1031,7 +1035,6 @@ ngx_quic_send_retry(ngx_connection_t *c,
pkt.odcid = inpkt->dcid;
pkt.dcid = inpkt->scid;

- /* TODO: generate routable dcid */
if (RAND_bytes(dcid, NGX_QUIC_SERVER_CID_LEN) != 1) {
return NGX_ERROR;
}
diff --git a/src/event/quic/ngx_event_quic_socket.c b/src/event/quic/ngx_event_quic_socket.c
--- a/src/event/quic/ngx_event_quic_socket.c
+++ b/src/event/quic/ngx_event_quic_socket.c
@@ -109,6 +109,13 @@ ngx_quic_open_sockets(ngx_connection_t *
failed:

ngx_rbtree_delete(&c->listening->rbtree, &qsock->udp.node);
+
+#if (NGX_QUIC_BPF)
+ if (ngx_quic_bpf_delete(c, qc, qsock) != NGX_OK) {
+ return NGX_ERROR;
+ }
+#endif
+
c->udp = NULL;

return NGX_ERROR;
@@ -160,6 +167,13 @@ ngx_quic_close_socket(ngx_connection_t *
ngx_queue_insert_head(&qc->free_sockets, &qsock->queue);

ngx_rbtree_delete(&c->listening->rbtree, &qsock->udp.node);
+
+#if (NGX_QUIC_BPF)
+ if (ngx_quic_bpf_delete(c, qc, qsock) != NGX_OK) {
+ return;
+ }
+#endif
+
qc->nsockets--;

ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
@@ -185,6 +199,12 @@ ngx_quic_listen(ngx_connection_t *c, ngx

ngx_rbtree_insert(&c->listening->rbtree, &qsock->udp.node);

+#if (NGX_QUIC_BPF)
+ if (ngx_quic_bpf_insert(c, qc, qsock) != NGX_OK) {
+ return NGX_ERROR;
+ }
+#endif
+
ngx_queue_insert_tail(&qc->sockets, &qsock->queue);

qc->nsockets++;
diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c
--- a/src/event/quic/ngx_event_quic_udp.c
+++ b/src/event/quic/ngx_event_quic_udp.c
@@ -160,7 +160,7 @@ ngx_quic_recvmsg(ngx_event_t *ev)
c->log->handler = NULL;

ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
- "quic recvmsg: fd:%d n:%z", c->fd, n);
+ "quic recvmsg: fd:%d n:%z", lc->fd, n);

c->log->handler = handler;
}
@@ -193,12 +193,23 @@ ngx_quic_recvmsg(ngx_event_t *ev)
ngx_accept_disabled = ngx_cycle->connection_n / 8
- ngx_cycle->free_connection_n;

- c = ngx_get_connection(lc->fd, ev->log);
- if (c == NULL) {
+ c = NULL;
+
+#if (NGX_QUIC_BPF)
+ if (ngx_quic_bpf_get_client_connection(lc, &c) != NGX_OK) {
return;
}
+#endif

- c->shared = 1;
+ if (c == NULL) {
+ c = ngx_get_connection(lc->fd, ev->log);
+ if (c == NULL) {
+ return;
+ }
+
+ c->shared = 1;
+ }
+
c->type = SOCK_DGRAM;
c->socklen = socklen;

@@ -309,7 +320,7 @@ ngx_quic_recvmsg(ngx_event_t *ev)

ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0,
"*%uA quic recvmsg: %V fd:%d n:%z",
- c->number, &addr, c->fd, n);
+ c->number, &addr, lc->fd, n);
}

}
diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c
--- a/src/os/unix/ngx_process_cycle.c
+++ b/src/os/unix/ngx_process_cycle.c
@@ -955,7 +955,8 @@ ngx_worker_process_exit(ngx_cycle_t *cyc
&& c[i].read
&& !c[i].read->accept
&& !c[i].read->channel
- && !c[i].read->resolver)
+ && !c[i].read->resolver
+ && !c[i].read->quic)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"*%uA open socket #%d left in connection %ui",
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel