# HG changeset patch
# User Liu Qiao <qiao.liu@intel.com>
# Date 1599735293 14400
# Thu Sep 10 06:54:53 2020 -0400
# Node ID f79d524a2cc0093c53490f947564e42371cf944f
# Parent da5e3f5b16733167142b599b6af3ce9469a07d52
Use BPF to distribute packet to different work thread.
Use Berkeley Packet Filter to get packet queue_mapping number,
and use this queue_mapping number to distribute the packet to
different work thread, this will improve CPU utilization and http
latency.
Author: Samudrala, Sridhar <sridhar.samudrala@intel.com>

diff -r da5e3f5b1673 -r f79d524a2cc0 auto/os/linux
--- a/auto/os/linux Wed Sep 02 23:13:36 2020 +0300
+++ b/auto/os/linux Thu Sep 10 06:54:53 2020 -0400
@@ -32,6 +32,10 @@
have=NGX_HAVE_POSIX_FADVISE . auto/nohave
fi
+if [ $version -lt 263680 ]; then
+ have=NGX_HAVE_REUSEPORT_CBPF . auto/nohave
+fi
+
# epoll, EPOLLET version
ngx_feature="epoll"
diff -r da5e3f5b1673 -r f79d524a2cc0 auto/unix
--- a/auto/unix Wed Sep 02 23:13:36 2020 +0300
+++ b/auto/unix Thu Sep 10 06:54:53 2020 -0400
@@ -331,6 +331,17 @@
ngx_feature_test="setsockopt(0, SOL_SOCKET, SO_REUSEPORT, NULL, 0)"
.. auto/feature
+ngx_feature="SO_REUSEPORT_CBPF"
+ngx_feature_name="NGX_HAVE_REUSEPORT_CBPF"
+ngx_feature_run=no
+ngx_feature_incs="#include <sys/socket.h>
+ #include <linux/filter.h>
+ #include <error.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="setsockopt(0, SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF, NULL, 0)"
+. auto/feature
+
ngx_feature="SO_ACCEPTFILTER"
ngx_feature_name="NGX_HAVE_DEFERRED_ACCEPT"
diff -r da5e3f5b1673 -r f79d524a2cc0 src/core/ngx_connection.c
--- a/src/core/ngx_connection.c Wed Sep 02 23:13:36 2020 +0300
+++ b/src/core/ngx_connection.c Thu Sep 10 06:54:53 2020 -0400
@@ -8,7 +8,10 @@
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_event.h>
-
+#if (NGX_HAVE_REUSEPORT_CBPF)
+#include <linux/filter.h>
+#include <error.h>
+#endif
ngx_os_io_t ngx_io;
@@ -708,6 +711,35 @@
return NGX_OK;
}
+#if(NGX_HAVE_REUSEPORT)
+#if(NGX_HAVE_REUSEPORT_CBPF)
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
+#endif
+
+static void attach_bpf(int fd, uint16_t n)
+{
+ struct sock_filter code[] = {
+ /* A = skb->queue_mapping */
+ { BPF_LD | BPF_W | BPF_ABS, 0, 0, SKF_AD_OFF + SKF_AD_QUEUE },
+ /* A = A % n */
+ { BPF_ALU | BPF_MOD, 0, 0, n },
+ /* return A */
+ { BPF_RET | BPF_A, 0, 0, 0 },
+ };
+ struct sock_fprog p = {
+ .len = ARRAY_SIZE(code),
+ .filter = code,
+ };
+
+ if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF, &p, sizeof(p)))
+ printf("failed to set SO_ATTACH_REUSEPORT_CBPF");
+ else
+ printf("bpf prog attached to fd:%d\n", fd);
+}
+#endif
+#endif
+
void
ngx_configure_listening_sockets(ngx_cycle_t *cycle)
@@ -719,6 +751,11 @@
#if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER)
struct accept_filter_arg af;
#endif
+#if (NGX_HAVE_REUSEPORT)
+#if (NGX_HAVE_REUSEPORT_CBPF)
+ ngx_core_conf_t* ccf ;
+#endif
+#endif
ls = cycle->listening.elts;
for (i = 0; i < cycle->listening.nelts; i++) {
@@ -1011,6 +1048,16 @@
}
#endif
+#if (NGX_HAVE_REUSEPORT)
+#if (NGX_HAVE_REUSEPORT_CBPF)
+ if(ls[i].reuseport)
+ {
+ ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx,ngx_core_module);
+ if(ccf)
+ attach_bpf(ls[i].fd, ccf->worker_processes);
+ }
+#endif
+#endif
}
return;

_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel