Hi Team,

I used Nginx to do 443:443 reverse proxy with Mellanox Connect6 DX networking cards.
I can make KTLS work for Nginx, but cannot see KTLS offloading (inline TLS @ MLX6) working.
Please help on what I missed?
Many thanks,

Liwu
-----------------
To utilize Openssh 3.0 and Nginx 1.21.1: I followed this instruction:
https://www.nginx.com/blog/improving-nginx-performance-with-kernel-tls/
To enable MLX6 inline TLS I followed this instruction:
https://docs.nvidia.com/networking/display/OFEDv521040/Kernel+Transport+Layer+Security+(kTLS)+Offloads

Here are further system information:

root@r57-8814:/boot# nginx -V
nginx version: nginx/1.21.4
built by gcc 11.2.0 (Ubuntu 11.2.0-19ubuntu1)
built with OpenSSL 3.0.0 7 sep 2021
TLS SNI support enabled
configure arguments: --with-debug --prefix=/usr/local --conf-path=/usr/local/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --http-client-body-temp-path=/var/cache/nginx/client_temp --http-proxy-temp-path=/var/cache/nginx/proxy_temp --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp --http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx --with-compat --with-file-aio --with-threads --with-http_addition_module --with-http_auth_request_module --with-http_dav_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_mp4_module --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_ssl_module --with-http_stub_status_module --with-http_sub_module --with-http_v2_module --with-mail --with-mail_ssl_module --with-stream --with-stream_realip_module --with-stream_ssl_module --with-stream_ssl_preread_module --with-openssl=../openssl-3.0.0 --with-openssl-opt=enable-ktls --with-cc-opt='-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fPIC'


root@r57-8814:~# uname -a
Linux r57-8814 5.15.0-37-generic #39-Ubuntu SMP Wed Jun 1 19:16:45 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
root@r57-8814:~# ethtool -k enp202s0f0np0 |grep tls
tls-hw-tx-offload: on
tls-hw-rx-offload: on
tls-hw-record: off [fixed]
root@r57-8814:~# ethtool -k enp202s0f1np1 |grep tls
tls-hw-tx-offload: on
tls-hw-rx-offload: on
tls-hw-record: off [fixed]
root@r57-8814:~# lsmod |grep tls
tls 106496 77 mlx5_core
root@r57-8814:/boot# grep TLS config-5.15.0-37-generic
CONFIG_TLS=m
CONFIG_TLS_DEVICE=y
# CONFIG_TLS_TOE is not set
CONFIG_CHELSIO_TLS_DEVICE=m
CONFIG_MLX5_FPGA_TLS=y
CONFIG_MLX5_TLS=y
CONFIG_MLX5_EN_TLS=y
CONFIG_FB_TFT_TLS8204=m

root@r57-8814:/usr/local/etc/nginx# cat nginx.conf

#user nobody;
worker_processes 4;
worker_cpu_affinity 0001 0010 0100 1000;

#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;

#pid logs/nginx.pid;


events {
worker_connections 1024;
}


http {
include mime.types;
default_type application/octet-stream;

#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';

#access_log logs/access.log main;

sendfile on;
#tcp_nopush on;

#keepalive_timeout 0;
keepalive_timeout 65;

#gzip on;

upstream backend {
server 1.1.2.2:443;
server 1.1.2.3:443;
server 1.1.2.4:443;
server 1.1.2.5:443;
server 1.1.2.6:443;
server 1.1.2.7:443;
server 1.1.2.8:443;
server 1.1.2.9:443;
server 1.1.2.10:443;
}

server {
listen 443 ssl;
ssl_certificate /usr/local/etc/nginx/cert.crt;
ssl_certificate_key /usr/local/etc/nginx/cert.key;
ssl_session_cache builtin:1000 shared:SSL:10m;
ssl_conf_command Options KTLS;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;
ssl_prefer_server_ciphers on;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
location / {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# Fix the “It appears that your reverse proxy set up is broken" error.
proxy_pass https://backend;
proxy_ssl_certificate /usr/local/etc/nginx/cert.crt;
proxy_ssl_certificate_key /usr/local/etc/nginx/cert.key;
proxy_ssl_trusted_certificate /usr/local/etc/nginx/cert.crt;
proxy_ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
proxy_ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;
}
}

Though the following stats suggest the inline-TLS is not triggered.

root@r57-8814:/boot# ethtool -S enp202s0f1np1 |grep tls
tx_tls_encrypted_packets: 0
tx_tls_encrypted_bytes: 0
tx_tls_ooo: 0
tx_tls_dump_packets: 0
tx_tls_dump_bytes: 0
tx_tls_resync_bytes: 0
tx_tls_skip_no_sync_data: 0
tx_tls_drop_no_sync_data: 0
tx_tls_drop_bypass_req: 0
rx_tls_decrypted_packets: 0
rx_tls_decrypted_bytes: 0
rx_tls_resync_req_pkt: 0
rx_tls_resync_req_start: 0
rx_tls_resync_req_end: 0
rx_tls_resync_req_skip: 0
rx_tls_resync_res_ok: 0
rx_tls_resync_res_retry: 0
rx_tls_resync_res_skip: 0
rx_tls_err: 0
tx_tls_ctx: 0
tx_tls_del: 0
rx_tls_ctx: 0
rx_tls_del: 0

root@r57-8814:/boot# ethtool -S enp202s0f0np0 |grep tls
tx_tls_encrypted_packets: 0
tx_tls_encrypted_bytes: 0
tx_tls_ooo: 0
tx_tls_dump_packets: 0
tx_tls_dump_bytes: 0
tx_tls_resync_bytes: 0
tx_tls_skip_no_sync_data: 0
tx_tls_drop_no_sync_data: 0
tx_tls_drop_bypass_req: 0
rx_tls_decrypted_packets: 0
rx_tls_decrypted_bytes: 0
rx_tls_resync_req_pkt: 0
rx_tls_resync_req_start: 0
rx_tls_resync_req_end: 0
rx_tls_resync_req_skip: 0
rx_tls_resync_res_ok: 0
rx_tls_resync_res_retry: 0
rx_tls_resync_res_skip: 0
rx_tls_err: 0
tx_tls_ctx: 0
tx_tls_del: 0
rx_tls_ctx: 0
rx_tls_del: 0