On Thu, Sep 10, 2020 at 05:22:12AM +0000, Liu, Qiao wrote:
> # HG changeset patch
> # User Liu Qiao <qiao.liu@intel.com>
> # Date 1599735293 14400
> # Thu Sep 10 06:54:53 2020 -0400
> # Node ID f79d524a2cc0093c53490f947564e42371cf944f
> # Parent da5e3f5b16733167142b599b6af3ce9469a07d52
> Use BPF to distribute packet to different work thread.
> Use Berkeley Packet Filter to get packet queue_mapping number,
> and use this queue_mapping number to distribute the packet to
> different work thread, this will improve CPU utilization and http
> latency.
> Author: Samudrala, Sridhar <sridhar.samudrala@intel.com>

Do you have any real measurements with proper testing methodology?

Also, why not bind NIC queues to cores system-wide?

>
> diff -r da5e3f5b1673 -r f79d524a2cc0 auto/os/linux
> --- a/auto/os/linux Wed Sep 02 23:13:36 2020 +0300
> +++ b/auto/os/linux Thu Sep 10 06:54:53 2020 -0400
> @@ -32,6 +32,10 @@
> have=NGX_HAVE_POSIX_FADVISE . auto/nohave
> fi
> +if [ $version -lt 263680 ]; then
> + have=NGX_HAVE_REUSEPORT_CBPF . auto/nohave
> +fi
> +
> # epoll, EPOLLET version
> ngx_feature="epoll"
> diff -r da5e3f5b1673 -r f79d524a2cc0 auto/unix
> --- a/auto/unix Wed Sep 02 23:13:36 2020 +0300
> +++ b/auto/unix Thu Sep 10 06:54:53 2020 -0400
> @@ -331,6 +331,17 @@
> ngx_feature_test="setsockopt(0, SOL_SOCKET, SO_REUSEPORT, NULL, 0)"
> . auto/feature
> +ngx_feature="SO_REUSEPORT_CBPF"
> +ngx_feature_name="NGX_HAVE_REUSEPORT_CBPF"
> +ngx_feature_run=no
> +ngx_feature_incs="#include <sys/socket.h>
> + #include <linux/filter.h>
> + #include <error.h>"
> +ngx_feature_path=
> +ngx_feature_libs=
> +ngx_feature_test="setsockopt(0, SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF, NULL, 0)"
> +. auto/feature
> +
> ngx_feature="SO_ACCEPTFILTER"
> ngx_feature_name="NGX_HAVE_DEFERRED_ACCEPT"
> diff -r da5e3f5b1673 -r f79d524a2cc0 src/core/ngx_connection.c
> --- a/src/core/ngx_connection.c Wed Sep 02 23:13:36 2020 +0300
> +++ b/src/core/ngx_connection.c Thu Sep 10 06:54:53 2020 -0400
> @@ -8,7 +8,10 @@
> #include <ngx_config.h>
> #include <ngx_core.h>
> #include <ngx_event.h>
> -
> +#if (NGX_HAVE_REUSEPORT_CBPF)
> +#include <linux/filter.h>
> +#include <error.h>
> +#endif
> ngx_os_io_t ngx_io;
> @@ -708,6 +711,35 @@
> return NGX_OK;
> }
> +#if(NGX_HAVE_REUSEPORT)
> +#if(NGX_HAVE_REUSEPORT_CBPF)
> +#ifndef ARRAY_SIZE
> +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
> +#endif
> +
> +static void attach_bpf(int fd, uint16_t n)
> +{
> + struct sock_filter code[] = {
> + /* A = skb->queue_mapping */
> + { BPF_LD | BPF_W | BPF_ABS, 0, 0, SKF_AD_OFF + SKF_AD_QUEUE },
> + /* A = A % n */
> + { BPF_ALU | BPF_MOD, 0, 0, n },
> + /* return A */
> + { BPF_RET | BPF_A, 0, 0, 0 },
> + };
> + struct sock_fprog p = {
> + .len = ARRAY_SIZE(code),
> + .filter = code,
> + };
> +
> + if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_CBPF, &p, sizeof(p)))
> + printf("failed to set SO_ATTACH_REUSEPORT_CBPF");
> + else
> + printf("bpf prog attached to fd:%d\n", fd);
> +}
> +#endif
> +#endif
> +
> void
> ngx_configure_listening_sockets(ngx_cycle_t *cycle)
> @@ -719,6 +751,11 @@
> #if (NGX_HAVE_DEFERRED_ACCEPT && defined SO_ACCEPTFILTER)
> struct accept_filter_arg af;
> #endif
> +#if (NGX_HAVE_REUSEPORT)
> +#if (NGX_HAVE_REUSEPORT_CBPF)
> + ngx_core_conf_t* ccf ;
> +#endif
> +#endif
> ls = cycle->listening.elts;
> for (i = 0; i < cycle->listening.nelts; i++) {
> @@ -1011,6 +1048,16 @@
> }
> #endif
> +#if (NGX_HAVE_REUSEPORT)
> +#if (NGX_HAVE_REUSEPORT_CBPF)
> + if(ls[i].reuseport)
> + {
> + ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx,ngx_core_module);
> + if(ccf)
> + attach_bpf(ls[i].fd, ccf->worker_processes);
> + }
> +#endif
> +#endif
> }
> return;
>

> _______________________________________________
> nginx-devel mailing list
> nginx-devel@nginx.org
> http://mailman.nginx.org/mailman/listinfo/nginx-devel

_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel