# HG changeset patch
# User ben ben ishay <benishay@mellanox.com>
# Date 1554896607 -10800
# Wed Apr 10 14:43:27 2019 +0300
# Node ID 87938decdb98bf4a06ed18002a15156a5e8fbd67
# Parent 65074e13f1716e09c28d730586babad7930b7a98
Add support for using sendfile when openssl support ktls

when we need to transfer data between file and socket we prefer to use sendfile instead of write because we save the copy to a buffer.
the use of sendfile is possible in openssl only if it support ktls(the master of openssl support ktls) otherwise there is a copy of the data to userspace for encryption in any case (this paper explain this https://netdevconf.org/1.2/papers/ktls.pdf ).
the patch change the flow when the request is to send data over ssl and also the nginx use openssl that support ktls, the new flow using the sendfile function that tcp use for send data (ngx_linux_sendfile_chain).
the performence with this patch applied was check with apib benchmark(https://github.com/apigee/apib), one machine run nginx and the other machine that connect back to back to the first one run apib with this comand: ./apib -c <num of connection> -d 30 https://<ip address>/<file name to send>.
the file size was 100K.

the result display in this table , each value represnt average throughput in GBps of 10 runs.

num of connection | regular nginx | new nginx
1 5 5.2
2 7.5 8.5
3 7.7 9

this result prove that this patch increase nginx performance and thus is useful.

diff -r 65074e13f171 -r 87938decdb98 auto/feature
--- a/auto/feature Tue Mar 26 09:33:57 2019 +0300
+++ b/auto/feature Wed Apr 10 14:43:27 2019 +0300
@@ -41,6 +41,10 @@

ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS $ngx_feature_inc_path \
-o $NGX_AUTOTEST $NGX_AUTOTEST.c $NGX_TEST_LD_OPT $ngx_feature_libs"
+if [ "$ngx_feature_name" == "NGX_OPENSSL_KTLS" ];then
+ ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS $ngx_feature_inc_path -I$OPENSSL/include \
+ -o $NGX_AUTOTEST $NGX_AUTOTEST.c $NGX_TEST_LD_OPT $ngx_feature_libs"
+fi

ngx_feature_inc_path=

diff -r 65074e13f171 -r 87938decdb98 auto/lib/openssl/conf
--- a/auto/lib/openssl/conf Tue Mar 26 09:33:57 2019 +0300
+++ b/auto/lib/openssl/conf Wed Apr 10 14:43:27 2019 +0300
@@ -140,3 +140,12 @@
fi

fi
+ngx_feature="OpenSSL library with KTLS"
+ngx_feature_name="NGX_OPENSSL_KTLS"
+ngx_feature_run=no
+ngx_feature_incs="#include \"openssl/bio.h\" "
+ngx_feature_path=
+ngx_feature_libs="-lssl -lcrypto $NGX_LIBDL $NGX_LIBPTHREAD"
+ngx_feature_test="BIO_get_ktls_send(NULL)"
+. auto/feature
+
diff -r 65074e13f171 -r 87938decdb98 src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c Tue Mar 26 09:33:57 2019 +0300
+++ b/src/event/ngx_event_openssl.c Wed Apr 10 14:43:27 2019 +0300
@@ -1528,6 +1528,9 @@
#endif

sc->connection = SSL_new(ssl->ctx);
+#if (NGX_OPENSSL_KTLS)
+ sc->ktls = 0;
+#endif

if (sc->connection == NULL) {
ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, "SSL_new() failed");
@@ -1639,6 +1642,12 @@
c->recv_chain = ngx_ssl_recv_chain;
c->send_chain = ngx_ssl_send_chain;

+#if (NGX_OPENSSL_KTLS)
+ if(BIO_get_ktls_send(SSL_get_wbio(c->ssl->connection))){
+ c->ssl->ktls = 1;
+ c->send_chain = ngx_linux_sendfile_chain;
+ }
+#endif
#ifndef SSL_OP_NO_RENEGOTIATION
#if OPENSSL_VERSION_NUMBER < 0x10100000L
#ifdef SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS
diff -r 65074e13f171 -r 87938decdb98 src/event/ngx_event_openssl.h
--- a/src/event/ngx_event_openssl.h Tue Mar 26 09:33:57 2019 +0300
+++ b/src/event/ngx_event_openssl.h Wed Apr 10 14:43:27 2019 +0300
@@ -99,6 +99,9 @@
unsigned in_early:1;
unsigned early_preread:1;
unsigned write_blocked:1;
+#if (NGX_OPENSSL_KTLS)
+ unsigned ktls:1;
+#endif
};


diff -r 65074e13f171 -r 87938decdb98 src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c Tue Mar 26 09:33:57 2019 +0300
+++ b/src/http/ngx_http_request.c Wed Apr 10 14:43:27 2019 +0300
@@ -604,9 +604,15 @@
}

#if (NGX_HTTP_SSL)
- if (c->ssl) {
+#ifndef NGX_OPENSSL_KTLS
+ if (c->ssl){
r->main_filter_need_in_memory = 1;
}
+#else
+ if(!c->ssl->ktls && c->ssl){
+ r->main_filter_need_in_memory = 1;
+ }
+#endif
#endif

r->main = r;
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel