# HG changeset patch
# User muradm <mail@muradm.net>
# Date 1610902507 -10800
# Sun Jan 17 19:55:07 2021 +0300
# Node ID 74562c5f22f8e03be55d412649c00d1ef2ca6811
# Parent 83c4622053b02821a12d522d08eaff3ac27e65e3
Mail: added PROXY PROTOCOL support.

This implements propxy protocol support for both upstream and downstream.

Downstream proxy protocol support:

mail {
server {
listen <port> [ssl] proxy_protocol;
protocol <imap|pop3|smtp>;
}
}

This will properly handle incoming connections from load balancer sending
PROXY protocol header. Without this, it is impossible to run nginx mail
proxy behind such balancer. Header reading is done with existing function
"ngx_proxy_protocol_read", so it should support both v1 and v2 headers.
This will also set "sockaddr" and "local_sockaddr" addresses from received
header, mimicing "set_realip". While "realip_module" deals with variables
etc., which is necessary for HTTP protocol, mail protocols are pretty
strict, so there is no need for flexible handling of real addresses
received.

Upstream proxy protocol support:

mail {
server {
listen <port> [ssl];
protocol <imap|pop3|smtp>;
proxy_protocol on;
}
}

With this, upstream server (like Postfix, Exim, Dovecot) will have PROXY
protocol header. Mentioned programs do support proxy protocol out of the
box. Header is written with existing function "ngx_proxy_protocol_write"
which supports only v1 header writing. Contents of header are written
from "sockaddr" and "local_sockaddr".

Downstream and upstream proxy protocol support:

mail {
server {
listen <port> [ssl] proxy_protocol;
protocol <imap|pop3|smtp>;
proxy_protocol on;
}
}

This will combine both receiving PROXY header and sending PROXY header. With
this, upstream server (like Postfix, Exim, Dovecot) will receive the same
header as was sent by downstream load balancer.

Above configurations work for SSL as well and should be transparent to other
mail related configurations.

Added "connect_timeout".

diff -r 83c4622053b0 -r 74562c5f22f8 src/mail/ngx_mail.c
--- a/src/mail/ngx_mail.c Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail.c Sun Jan 17 19:55:07 2021 +0300
@@ -402,6 +402,7 @@
addrs[i].addr = sin->sin_addr.s_addr;

addrs[i].conf.ctx = addr[i].opt.ctx;
+ addrs[i].conf.proxy_protocol = addr[i].opt.proxy_protocol;
#if (NGX_MAIL_SSL)
addrs[i].conf.ssl = addr[i].opt.ssl;
#endif
@@ -436,6 +437,7 @@
addrs6[i].addr6 = sin6->sin6_addr;

addrs6[i].conf.ctx = addr[i].opt.ctx;
+ addrs6[i].conf.proxy_protocol = addr[i].opt.proxy_protocol;
#if (NGX_MAIL_SSL)
addrs6[i].conf.ssl = addr[i].opt.ssl;
#endif
diff -r 83c4622053b0 -r 74562c5f22f8 src/mail/ngx_mail.h
--- a/src/mail/ngx_mail.h Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail.h Sun Jan 17 19:55:07 2021 +0300
@@ -37,6 +37,7 @@
unsigned bind:1;
unsigned wildcard:1;
unsigned ssl:1;
+ unsigned proxy_protocol:1;
#if (NGX_HAVE_INET6)
unsigned ipv6only:1;
#endif
@@ -56,6 +57,7 @@
ngx_mail_conf_ctx_t *ctx;
ngx_str_t addr_text;
ngx_uint_t ssl; /* unsigned ssl:1; */
+ unsigned proxy_protocol:1;
} ngx_mail_addr_conf_t;

typedef struct {
@@ -190,6 +192,7 @@
void **ctx;
void **main_conf;
void **srv_conf;
+ ngx_mail_addr_conf_t *addr_conf;

ngx_resolver_ctx_t *resolver_ctx;

@@ -197,6 +200,7 @@

ngx_uint_t mail_state;

+ unsigned proxy_protocol:1;
unsigned protocol:3;
unsigned blocked:1;
unsigned quit:1;
diff -r 83c4622053b0 -r 74562c5f22f8 src/mail/ngx_mail_core_module.c
--- a/src/mail/ngx_mail_core_module.c Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail_core_module.c Sun Jan 17 19:55:07 2021 +0300
@@ -548,6 +548,11 @@
#endif
}

+ if (ngx_strcmp(value[i].data, "proxy_protocol") == 0) {
+ ls->proxy_protocol = 1;
+ continue;
+ }
+
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"the invalid \"%V\" parameter", &value[i]);
return NGX_CONF_ERROR;
diff -r 83c4622053b0 -r 74562c5f22f8 src/mail/ngx_mail_handler.c
--- a/src/mail/ngx_mail_handler.c Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail_handler.c Sun Jan 17 19:55:07 2021 +0300
@@ -12,6 +12,8 @@


static void ngx_mail_init_session(ngx_connection_t *c);
+static void ngx_mail_init_connection_complete(ngx_connection_t *c);
+static void ngx_mail_proxy_protocol_handler(ngx_event_t *rev);

#if (NGX_MAIL_SSL)
static void ngx_mail_ssl_init_connection(ngx_ssl_t *ssl, ngx_connection_t *c);
@@ -128,6 +130,7 @@

s->main_conf = addr_conf->ctx->main_conf;
s->srv_conf = addr_conf->ctx->srv_conf;
+ s->addr_conf = addr_conf;

s->addr_text = &addr_conf->addr_text;

@@ -159,13 +162,161 @@

c->log_error = NGX_ERROR_INFO;

+ /*
+ * Before all process proxy protocol
+ */
+
+ if (addr_conf->proxy_protocol) {
+ s->proxy_protocol = 1;
+ c->log->action = "reading PROXY protocol header";
+ c->read->handler = ngx_mail_proxy_protocol_handler;
+
+ ngx_add_timer(c->read, cscf->timeout);
+
+ if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+ ngx_mail_close_connection(c);
+ }
+
+ return;
+ }
+
+ ngx_mail_init_connection_complete(c);
+}
+
+
+ngx_int_t
+ngx_mail_proxy_protoco_set_addrs(ngx_connection_t *c)
+{
+ ngx_addr_t addr_peer, addr_local;
+ u_char *p, text[NGX_SOCKADDR_STRLEN];
+ size_t len;
+
+ if (ngx_parse_addr(c->pool, &addr_peer,
+ c->proxy_protocol->src_addr.data,
+ c->proxy_protocol->src_addr.len) != NGX_OK)
+ {
+ return NGX_ERROR;
+ }
+
+ ngx_inet_set_port(addr_peer.sockaddr, c->proxy_protocol->src_port);
+
+ if (ngx_parse_addr(c->pool, &addr_local,
+ c->proxy_protocol->dst_addr.data,
+ c->proxy_protocol->dst_addr.len) != NGX_OK)
+ {
+ return NGX_ERROR;
+ }
+
+ ngx_inet_set_port(addr_local.sockaddr, c->proxy_protocol->dst_port);
+
+ len = ngx_sock_ntop(addr_peer.sockaddr, addr_peer.socklen, text,
+ NGX_SOCKADDR_STRLEN, 0);
+ if (len == 0) {
+ return NGX_ERROR;
+ }
+
+ p = ngx_pnalloc(c->pool, len);
+ if (p == NULL) {
+ return NGX_ERROR;
+ }
+
+ ngx_memcpy(p, text, len);
+
+ c->sockaddr = addr_peer.sockaddr;
+ c->socklen = addr_peer.socklen;
+ c->addr_text.len = len;
+ c->addr_text.data = p;
+
+ len = ngx_sock_ntop(addr_local.sockaddr, addr_local.socklen, text,
+ NGX_SOCKADDR_STRLEN, 0);
+ if (len == 0) {
+ return NGX_ERROR;
+ }
+
+ p = ngx_pnalloc(c->pool, len);
+ if (p == NULL) {
+ return NGX_ERROR;
+ }
+
+ ngx_memcpy(p, text, len);
+
+ c->local_sockaddr = addr_local.sockaddr;
+ c->local_socklen = addr_local.socklen;
+
+ return NGX_OK;
+}
+
+
+void
+ngx_mail_proxy_protocol_handler(ngx_event_t *rev)
+{
+ ngx_connection_t *c;
+ u_char *p, buf[NGX_PROXY_PROTOCOL_MAX_HEADER];
+ size_t size;
+ ssize_t n;
+
+ c = rev->data;
+
+ if (rev->timedout) {
+ ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT,
+ "mail PROXY protocol header timed out");
+ c->timedout = 1;
+ ngx_mail_close_connection(c);
+ return;
+ }
+
+ ngx_log_debug0(NGX_LOG_DEBUG_MAIL, c->log, 0,
+ "mail PROXY protocol handler");
+
+ size = NGX_PROXY_PROTOCOL_MAX_HEADER;
+
+ n = recv(c->fd, (char *) buf, size, MSG_PEEK);
+
+ ngx_log_debug1(NGX_LOG_DEBUG, c->log, 0, "mail recv(): %z", n);
+
+ p = ngx_proxy_protocol_read(c, buf, buf + n);
+
+ if (p == NULL) {
+ ngx_mail_close_connection(c);
+ return;
+ }
+
+ ngx_log_error(NGX_LOG_NOTICE, c->log, 0,
+ "PROXY protocol %V:%d => %V:%d",
+ &c->proxy_protocol->src_addr,
+ c->proxy_protocol->src_port,
+ &c->proxy_protocol->dst_addr,
+ c->proxy_protocol->dst_port);
+
+ size = p - buf;
+
+ if (c->recv(c, buf, size) != (ssize_t) size) {
+ ngx_mail_close_connection(c);
+ return;
+ }
+
+ if (ngx_mail_proxy_protoco_set_addrs(c) != NGX_OK) {
+ ngx_mail_close_connection(c);
+ return;
+ }
+
+ ngx_mail_init_connection_complete(c);
+}
+
+
+void
+ngx_mail_init_connection_complete(ngx_connection_t *c)
+{
#if (NGX_MAIL_SSL)
{
- ngx_mail_ssl_conf_t *sslcf;
+ ngx_mail_session_t *s;
+ ngx_mail_ssl_conf_t *sslcf;
+
+ s = c->data;

sslcf = ngx_mail_get_module_srv_conf(s, ngx_mail_ssl_module);

- if (sslcf->enable || addr_conf->ssl) {
+ if (sslcf->enable || s->addr_conf->ssl) {
c->log->action = "SSL handshaking";

ngx_mail_ssl_init_connection(&sslcf->ssl, c);
@@ -348,6 +499,7 @@
return;
}

+ c->log->action = "sending client greeting line";
c->write->handler = ngx_mail_send;

cscf->protocol->init_session(s, c);
diff -r 83c4622053b0 -r 74562c5f22f8 src/mail/ngx_mail_proxy_module.c
--- a/src/mail/ngx_mail_proxy_module.c Tue Jan 12 16:59:31 2021 +0300
+++ b/src/mail/ngx_mail_proxy_module.c Sun Jan 17 19:55:07 2021 +0300
@@ -19,6 +19,8 @@
ngx_flag_t smtp_auth;
size_t buffer_size;
ngx_msec_t timeout;
+ ngx_msec_t connect_timeout;
+ ngx_flag_t proxy_protocol;
} ngx_mail_proxy_conf_t;


@@ -36,7 +38,9 @@
static void *ngx_mail_proxy_create_conf(ngx_conf_t *cf);
static char *ngx_mail_proxy_merge_conf(ngx_conf_t *cf, void *parent,
void *child);
-
+static void ngx_mail_proxy_connect_handler(ngx_event_t *ev);
+static void ngx_mail_proxy_start(ngx_mail_session_t *s);
+static void ngx_mail_proxy_send_proxy_protocol(ngx_mail_session_t *s);

static ngx_command_t ngx_mail_proxy_commands[] = {

@@ -61,6 +65,13 @@
offsetof(ngx_mail_proxy_conf_t, timeout),
NULL },

+ { ngx_string("connect_timeout"),
+ NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_msec_slot,
+ NGX_MAIL_SRV_CONF_OFFSET,
+ offsetof(ngx_mail_proxy_conf_t, connect_timeout),
+ NULL },
+
{ ngx_string("proxy_pass_error_message"),
NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_FLAG,
ngx_conf_set_flag_slot,
@@ -82,6 +93,13 @@
offsetof(ngx_mail_proxy_conf_t, smtp_auth),
NULL },

+ { ngx_string("proxy_protocol"),
+ NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_MAIL_SRV_CONF_OFFSET,
+ offsetof(ngx_mail_proxy_conf_t, proxy_protocol),
+ NULL },
+
ngx_null_command
};

@@ -156,7 +174,6 @@
p->upstream.connection->pool = s->connection->pool;

s->connection->read->handler = ngx_mail_proxy_block_read;
- p->upstream.connection->write->handler = ngx_mail_proxy_dummy_handler;

pcf = ngx_mail_get_module_srv_conf(s, ngx_mail_proxy_module);

@@ -169,23 +186,139 @@

s->out.len = 0;

+ if (rc == NGX_AGAIN) {
+ p->upstream.connection->write->handler = ngx_mail_proxy_connect_handler;
+ p->upstream.connection->read->handler = ngx_mail_proxy_connect_handler;
+
+ ngx_add_timer(p->upstream.connection->write, pcf->connect_timeout);
+
+ ngx_log_debug0(NGX_LOG_DEBUG_MAIL, s->connection->log, 0, "mail proxy delay connect");
+ return;
+ }
+
+ if (pcf->proxy_protocol) {
+ ngx_mail_proxy_send_proxy_protocol(s);
+ return;
+ }
+
+ ngx_mail_proxy_start(s);
+}
+
+
+void
+ngx_mail_proxy_connect_handler(ngx_event_t *ev)
+{
+ ngx_connection_t *c;
+ ngx_mail_session_t *s;
+ ngx_mail_proxy_conf_t *pcf;
+
+ c = ev->data;
+ s = c->data;
+
+ if (ev->timedout) {
+ ngx_log_error(NGX_LOG_ERR, c->log, NGX_ETIMEDOUT, "upstream timed out");
+ ngx_mail_session_internal_server_error(s);
+ return;
+ }
+
+ ngx_del_timer(c->write);
+
+ ngx_log_debug0(NGX_LOG_DEBUG_MAIL, c->log, 0,
+ "mail proxy connect upstream");
+
+ pcf = ngx_mail_get_module_srv_conf(s, ngx_mail_proxy_module);
+
+ if (pcf->proxy_protocol) {
+ ngx_mail_proxy_send_proxy_protocol(s);
+ return;
+ }
+
+ ngx_mail_proxy_start(s);
+}
+
+
+void
+ngx_mail_proxy_start(ngx_mail_session_t *s)
+{
+ ngx_connection_t *pc;
+
+ pc = s->proxy->upstream.connection;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_MAIL, s->connection->log, 0,
+ "mail proxy starting");
+
+ pc->write->handler = ngx_mail_proxy_dummy_handler;
+
switch (s->protocol) {

case NGX_MAIL_POP3_PROTOCOL:
- p->upstream.connection->read->handler = ngx_mail_proxy_pop3_handler;
+ pc->read->handler = ngx_mail_proxy_pop3_handler;
s->mail_state = ngx_pop3_start;
break;

case NGX_MAIL_IMAP_PROTOCOL:
- p->upstream.connection->read->handler = ngx_mail_proxy_imap_handler;
+ pc->read->handler = ngx_mail_proxy_imap_handler;
s->mail_state = ngx_imap_start;
break;

default: /* NGX_MAIL_SMTP_PROTOCOL */
- p->upstream.connection->read->handler = ngx_mail_proxy_smtp_handler;
+ pc->read->handler = ngx_mail_proxy_smtp_handler;
s->mail_state = ngx_smtp_start;
break;
}
+
+ if (pc->read->ready) {
+ ngx_post_event(pc->read, &ngx_posted_events);
+ }
+}
+
+
+void
+ngx_mail_proxy_send_proxy_protocol(ngx_mail_session_t *s)
+{
+ u_char *p;
+ ssize_t n, size;
+ ngx_connection_t *c, *pc;
+ ngx_peer_connection_t *u;
+ u_char buf[NGX_PROXY_PROTOCOL_MAX_HEADER];
+
+ c = s->connection;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_MAIL, c->log, 0,
+ "mail proxy send PROXY protocol header");
+
+ p = ngx_proxy_protocol_write(c, buf, buf + NGX_PROXY_PROTOCOL_MAX_HEADER);
+ if (p == NULL) {
+ ngx_mail_proxy_internal_server_error(s);
+ return;
+ }
+
+ u = &s->proxy->upstream;
+
+ pc = u->connection;
+
+ size = p - buf;
+
+ n = pc->send(pc, buf, size);
+
+ if (n != size) {
+
+ /*
+ * PROXY protocol specification:
+ * The sender must always ensure that the header
+ * is sent at once, so that the transport layer
+ * maintains atomicity along the path to the receiver.
+ */
+
+ ngx_log_error(NGX_LOG_ERR, c->log, 0,
+ "could not send PROXY protocol header at once (%z)", n);
+
+ ngx_mail_proxy_internal_server_error(s);
+
+ return;
+ }
+
+ ngx_mail_proxy_start(s);
}


@@ -1184,6 +1317,8 @@
pcf->smtp_auth = NGX_CONF_UNSET;
pcf->buffer_size = NGX_CONF_UNSET_SIZE;
pcf->timeout = NGX_CONF_UNSET_MSEC;
+ pcf->connect_timeout = NGX_CONF_UNSET_MSEC;
+ pcf->proxy_protocol = NGX_CONF_UNSET;

return pcf;
}
@@ -1202,6 +1337,8 @@
ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size,
(size_t) ngx_pagesize);
ngx_conf_merge_msec_value(conf->timeout, prev->timeout, 24 * 60 * 60000);
+ ngx_conf_merge_msec_value(conf->connect_timeout, prev->connect_timeout, 1000);
+ ngx_conf_merge_value(conf->proxy_protocol, prev->proxy_protocol, 0);

return NGX_CONF_OK;
}
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel