> On 10 Nov 2023, at 14:07, Roman Arutyunyan <arut@nginx.com> wrote:
>
> # HG changeset patch
> # User Roman Arutyunyan <arut@nginx.com>
> # Date 1699543504 -14400
> # Thu Nov 09 19:25:04 2023 +0400
> # Node ID 3cab85fe55272835674b7f1c296796955256d019
> # Parent 1d3464283405a4d8ac54caae9bf1815c723f04c5
> Stream: ngx_stream_pass_module.
>
> The module allows to pass connections from Stream to other modules such as HTTP
> or Mail, as well as back to Stream. Previously, this was only possible with
> proxying. Connections with preread buffer read out from socket cannot be
> passed.
>
> The module allows to terminate SSL selectively based on SNI.

Note that terminating SSL before passing connections may be challenging
and requires appropriate preparations in the Stream module itself.
Notably, Stream unconditionally disables SSL buffering now,
which may affect performance in the modules being passed.
The following patches are to address such deficiency (and effectively,
they are used to catch up differences with the http module).

# HG changeset patch
# User Sergey Kandaurov <pluknet@nginx.com>
# Date 1702653986 -14400
# Fri Dec 15 19:26:26 2023 +0400
# Node ID aed7c0875fc690ccaa7288f66624ddca8299bff5
# Parent 219662ea1613ab68d4d5d4085394bba75993ae42
SSL: make it possible to disable SSL buffering.

It is now disabled if "ssl_buffer_size 0;" is used in the configuration.
Previously, such configuration was meaningless and resulted in CPU hog.

diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -1709,7 +1709,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl
return NGX_ERROR;
}

- sc->buffer = ((flags & NGX_SSL_BUFFER) != 0);
+ sc->buffer = ((flags & NGX_SSL_BUFFER) != 0) && (ssl->buffer_size != 0);
sc->buffer_size = ssl->buffer_size;

sc->session_ctx = ssl->ctx;
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -947,6 +947,7 @@ ngx_http_ssl_servername(ngx_ssl_conn_t *

sscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_ssl_module);

+ c->ssl->buffer = (sscf->buffer_size != 0);
c->ssl->buffer_size = sscf->buffer_size;

if (sscf->ssl.ctx) {
# HG changeset patch
# User Sergey Kandaurov <pluknet@nginx.com>
# Date 1702653990 -14400
# Fri Dec 15 19:26:30 2023 +0400
# Node ID 272afdf3fe232dd9770c8644a5230830192b434d
# Parent aed7c0875fc690ccaa7288f66624ddca8299bff5
Stream: the "ssl_buffer_size" directive.

This change introduces behaviour change in SSL buffering, which is now
enabled by default. It can be disabled with "ssl_buffer_size 0;".
Previously, it was unconditionally disabled.

In particular, it is useful when Stream passes SSL connections after SSL
termination to other modules using ngx_stream_pass_module.

diff --git a/src/stream/ngx_stream_ssl_module.c b/src/stream/ngx_stream_ssl_module.c
--- a/src/stream/ngx_stream_ssl_module.c
+++ b/src/stream/ngx_stream_ssl_module.c
@@ -142,6 +142,13 @@ static ngx_command_t ngx_stream_ssl_com
offsetof(ngx_stream_ssl_srv_conf_t, ciphers),
NULL },

+ { ngx_string("ssl_buffer_size"),
+ NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_size_slot,
+ NGX_STREAM_SRV_CONF_OFFSET,
+ offsetof(ngx_stream_ssl_srv_conf_t, buffer_size),
+ NULL },
+
{ ngx_string("ssl_verify_client"),
NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1,
ngx_conf_set_enum_slot,
@@ -414,7 +421,7 @@ ngx_stream_ssl_init_connection(ngx_ssl_t
return NGX_ERROR;
}

- if (ngx_ssl_create_connection(ssl, c, 0) != NGX_OK) {
+ if (ngx_ssl_create_connection(ssl, c, NGX_SSL_BUFFER) != NGX_OK) {
return NGX_ERROR;
}

@@ -527,6 +534,9 @@ ngx_stream_ssl_servername(ngx_ssl_conn_t

sscf = ngx_stream_get_module_srv_conf(s, ngx_stream_ssl_module);

+ c->ssl->buffer = (sscf->buffer_size != 0);
+ c->ssl->buffer_size = sscf->buffer_size;
+
if (sscf->ssl.ctx) {
if (SSL_set_SSL_CTX(ssl_conn, sscf->ssl.ctx) == NULL) {
goto error;
@@ -786,6 +796,7 @@ ngx_stream_ssl_create_srv_conf(ngx_conf_
sscf->conf_commands = NGX_CONF_UNSET_PTR;
sscf->prefer_server_ciphers = NGX_CONF_UNSET;
sscf->reject_handshake = NGX_CONF_UNSET;
+ sscf->buffer_size = NGX_CONF_UNSET_SIZE;
sscf->verify = NGX_CONF_UNSET_UINT;
sscf->verify_depth = NGX_CONF_UNSET_UINT;
sscf->builtin_session_cache = NGX_CONF_UNSET;
@@ -821,6 +832,9 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t
|NGX_SSL_TLSv1|NGX_SSL_TLSv1_1
|NGX_SSL_TLSv1_2|NGX_SSL_TLSv1_3));

+ ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size,
+ NGX_SSL_BUFSIZE);
+
ngx_conf_merge_uint_value(conf->verify, prev->verify, 0);
ngx_conf_merge_uint_value(conf->verify_depth, prev->verify_depth, 1);

@@ -930,6 +944,8 @@ ngx_stream_ssl_merge_srv_conf(ngx_conf_t
}
}

+ conf->ssl.buffer_size = conf->buffer_size;
+
if (conf->verify) {

if (conf->client_certificate.len == 0 && conf->verify != 3) {
diff --git a/src/stream/ngx_stream_ssl_module.h b/src/stream/ngx_stream_ssl_module.h
--- a/src/stream/ngx_stream_ssl_module.h
+++ b/src/stream/ngx_stream_ssl_module.h
@@ -27,6 +27,8 @@ typedef struct {
ngx_uint_t verify;
ngx_uint_t verify_depth;

+ size_t buffer_size;
+
ssize_t builtin_session_cache;

time_t session_timeout;


>
> stream {
> server {
> listen 8000 default_server;
> ssl_preread on;
> ...
> }
>
> server {
> listen 8000;
> server_name foo.example.com;
> pass 8001; # to HTTP
> }
>
> server {
> listen 8000;
> server_name bar.example.com;
> ...
> }
> }
>
> http {
> server {
> listen 8001 ssl;
> ...
>
> location / {
> root html;
> }
> }
> }
>
> diff --git a/auto/modules b/auto/modules
> --- a/auto/modules
> +++ b/auto/modules
> @@ -1166,6 +1166,16 @@ if [ $STREAM != NO ]; then
> . auto/module
> fi
>
> + if [ $STREAM_PASS = YES ]; then
> + ngx_module_name=ngx_stream_pass_module
> + ngx_module_deps=
> + ngx_module_srcs=src/stream/ngx_stream_pass_module.c
> + ngx_module_libs=
> + ngx_module_link=$STREAM_PASS
> +
> + . auto/module
> + fi
> +
> if [ $STREAM_SET = YES ]; then
> ngx_module_name=ngx_stream_set_module
> ngx_module_deps=
> diff --git a/auto/options b/auto/options
> --- a/auto/options
> +++ b/auto/options
> @@ -127,6 +127,7 @@ STREAM_GEOIP=NO
> STREAM_MAP=YES
> STREAM_SPLIT_CLIENTS=YES
> STREAM_RETURN=YES
> +STREAM_PASS=YES
> STREAM_SET=YES
> STREAM_UPSTREAM_HASH=YES
> STREAM_UPSTREAM_LEAST_CONN=YES
> @@ -337,6 +338,7 @@ use the \"--with-mail_ssl_module\" optio
> --without-stream_split_clients_module)
> STREAM_SPLIT_CLIENTS=NO ;;
> --without-stream_return_module) STREAM_RETURN=NO ;;
> + --without-stream_pass_module) STREAM_PASS=NO ;;
> --without-stream_set_module) STREAM_SET=NO ;;
> --without-stream_upstream_hash_module)
> STREAM_UPSTREAM_HASH=NO ;;
> @@ -556,6 +558,7 @@ cat << END
> --without-stream_split_clients_module
> disable ngx_stream_split_clients_module
> --without-stream_return_module disable ngx_stream_return_module
> + --without-stream_pass_module disable ngx_stream_pass_module
> --without-stream_set_module disable ngx_stream_set_module
> --without-stream_upstream_hash_module
> disable ngx_stream_upstream_hash_module
> diff --git a/src/stream/ngx_stream_pass_module.c b/src/stream/ngx_stream_pass_module.c
> new file mode 100644
> --- /dev/null
> +++ b/src/stream/ngx_stream_pass_module.c
> @@ -0,0 +1,245 @@
> +
> +/*
> + * Copyright (C) Roman Arutyunyan
> + * Copyright (C) Nginx, Inc.
> + */
> +
> +
> +#include <ngx_config.h>
> +#include <ngx_core.h>
> +#include <ngx_stream.h>
> +
> +
> +typedef struct {
> + ngx_addr_t *addr;
> + ngx_stream_complex_value_t *addr_value;
> +} ngx_stream_pass_srv_conf_t;
> +
> +
> +static void ngx_stream_pass_handler(ngx_stream_session_t *s);
> +static void *ngx_stream_pass_create_srv_conf(ngx_conf_t *cf);
> +static char *ngx_stream_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
> +
> +
> +static ngx_command_t ngx_stream_pass_commands[] = {
> +
> + { ngx_string("pass"),
> + NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1,
> + ngx_stream_pass,
> + NGX_STREAM_SRV_CONF_OFFSET,
> + 0,
> + NULL },
> +
> + ngx_null_command
> +};
> +
> +
> +static ngx_stream_module_t ngx_stream_pass_module_ctx = {
> + NULL, /* preconfiguration */
> + NULL, /* postconfiguration */
> +
> + NULL, /* create main configuration */
> + NULL, /* init main configuration */
> +
> + ngx_stream_pass_create_srv_conf, /* create server configuration */
> + NULL /* merge server configuration */
> +};
> +
> +
> +ngx_module_t ngx_stream_pass_module = {
> + NGX_MODULE_V1,
> + &ngx_stream_pass_module_ctx, /* module conaddr */
> + ngx_stream_pass_commands, /* module directives */
> + NGX_STREAM_MODULE, /* module type */
> + NULL, /* init master */
> + NULL, /* init module */
> + NULL, /* init process */
> + NULL, /* init thread */
> + NULL, /* exit thread */
> + NULL, /* exit process */
> + NULL, /* exit master */
> + NGX_MODULE_V1_PADDING
> +};
> +
> +
> +static void
> +ngx_stream_pass_handler(ngx_stream_session_t *s)
> +{
> + ngx_url_t u;
> + ngx_str_t url;
> + ngx_addr_t *addr;
> + ngx_uint_t i;
> + ngx_listening_t *ls;
> + ngx_connection_t *c;
> + ngx_stream_pass_srv_conf_t *pscf;
> +
> + c = s->connection;
> +
> + c->log->action = "passing connection to another module";
> +
> + if (c->buffer && c->buffer->pos != c->buffer->last) {
> + ngx_log_error(NGX_LOG_ERR, s->connection->log, 0,
> + "cannot pass connection with preread data");
> + goto failed;
> + }
> +
> + pscf = ngx_stream_get_module_srv_conf(s, ngx_stream_pass_module);
> +
> + addr = pscf->addr;
> +
> + if (addr == NULL) {
> + if (ngx_stream_complex_value(s, pscf->addr_value, &url) != NGX_OK) {
> + goto failed;
> + }
> +
> + ngx_memzero(&u, sizeof(ngx_url_t));
> +
> + u.url = url;
> + u.listen = 1;
> + u.no_resolve = 1;
> +
> + if (ngx_parse_url(s->connection->pool, &u) != NGX_OK) {
> + if (u.err) {
> + ngx_log_error(NGX_LOG_ERR, s->connection->log, 0,
> + "%s in pass \"%V\"", u.err, &u.url);
> + }
> +
> + goto failed;
> + }
> +
> + if (u.naddrs == 0) {
> + ngx_log_error(NGX_LOG_ERR, s->connection->log, 0,
> + "no addresses in pass \"%V\"", &u.url);
> + goto failed;
> + }
> +
> + addr = &u.addrs[0];
> + }
> +
> + ngx_log_debug1(NGX_LOG_DEBUG_STREAM, c->log, 0,
> + "stream pass addr: \"%V\"", &addr->name);
> +
> + ls = ngx_cycle->listening.elts;
> +
> + for (i = 0; i < ngx_cycle->listening.nelts; i++) {
> + if (ngx_cmp_sockaddr(ls[i].sockaddr, ls[i].socklen,
> + addr->sockaddr, addr->socklen, 1)
> + == NGX_OK)
> + {
> + c->listening = &ls[i];
> +
> + c->data = NULL;
> + c->buffer = NULL;
> +
> + *c->log = c->listening->log;
> + c->log->handler = NULL;
> + c->log->data = NULL;
> +
> + c->listening->handler(c);
> +
> + return;
> + }
> + }
> +
> + ngx_log_error(NGX_LOG_ERR, c->log, 0,
> + "listen not found for \"%V\"", &addr->name);
> +
> + ngx_stream_finalize_session(s, NGX_STREAM_OK);
> +
> + return;
> +
> +failed:
> +
> + ngx_stream_finalize_session(s, NGX_STREAM_INTERNAL_SERVER_ERROR);
> +}
> +
> +
> +static void *
> +ngx_stream_pass_create_srv_conf(ngx_conf_t *cf)
> +{
> + ngx_stream_pass_srv_conf_t *conf;
> +
> + conf = ngx_pcalloc(cf->pool, sizeof(ngx_stream_pass_srv_conf_t));
> + if (conf == NULL) {
> + return NULL;
> + }
> +
> + /*
> + * set by ngx_pcalloc():
> + *
> + * conf->addr = NULL;
> + * conf->addr_value = NULL;
> + */
> +
> + return conf;
> +}
> +
> +
> +static char *
> +ngx_stream_pass(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
> +{
> + ngx_stream_pass_srv_conf_t *pscf = conf;
> +
> + ngx_url_t u;
> + ngx_str_t *value, *url;
> + ngx_stream_complex_value_t cv;
> + ngx_stream_core_srv_conf_t *cscf;
> + ngx_stream_compile_complex_value_t ccv;
> +
> + if (pscf->addr || pscf->addr_value) {
> + return "is duplicate";
> + }
> +
> + cscf = ngx_stream_conf_get_module_srv_conf(cf, ngx_stream_core_module);
> +
> + cscf->handler = ngx_stream_pass_handler;
> +
> + value = cf->args->elts;
> +
> + url = &value[1];
> +
> + ngx_memzero(&ccv, sizeof(ngx_stream_compile_complex_value_t));
> +
> + ccv.cf = cf;
> + ccv.value = url;
> + ccv.complex_value = &cv;
> +
> + if (ngx_stream_compile_complex_value(&ccv) != NGX_OK) {
> + return NGX_CONF_ERROR;
> + }
> +
> + if (cv.lengths) {
> + pscf->addr_value = ngx_palloc(cf->pool,
> + sizeof(ngx_stream_complex_value_t));
> + if (pscf->addr_value == NULL) {
> + return NGX_CONF_ERROR;
> + }
> +
> + *pscf->addr_value = cv;
> +
> + return NGX_CONF_OK;
> + }
> +
> + ngx_memzero(&u, sizeof(ngx_url_t));
> +
> + u.url = *url;
> + u.listen = 1;
> +
> + if (ngx_parse_url(cf->pool, &u) != NGX_OK) {
> + if (u.err) {
> + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
> + "%s in \"%V\" of the \"pass\" directive",
> + u.err, &u.url);
> + }
> +
> + return NGX_CONF_ERROR;
> + }
> +
> + if (u.naddrs == 0) {
> + return "has no addresses";
> + }
> +
> + pscf->addr = &u.addrs[0];
> +
> + return NGX_CONF_OK;
> +}
> _______________________________________________
> nginx-devel mailing list
> nginx-devel@nginx.org
> https://mailman.nginx.org/mailman/listinfo/nginx-devel

--
Sergey Kandaurov
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel