Hello!
On Fri, Jan 26, 2024 at 04:27:30PM +0400, Sergey Kandaurov wrote:
>
> > On 27 Nov 2023, at 06:50, Maxim Dounin <mdounin@mdounin.ru> wrote:
> >
> > # HG changeset patch
> > # User Maxim Dounin <mdounin@mdounin.ru>
> > # Date 1701050170 -10800
> > # Mon Nov 27 04:56:10 2023 +0300
> > # Node ID 00c3e7333145ddb5ea0eeaaa66b3d9c26973c9c2
> > # Parent 61d08e4cf97cc073200ec32fc6ada9a2d48ffe51
> > AIO operations now add timers (ticket #2162).
> >
> > Each AIO (thread IO) operation being run is now accompanied with 1-minute
> > timer. This timer prevents unexpected shutdown of the worker process while
> > an AIO operation is running, and logs an alert if the operation is running
> > for too long.
> >
> > This fixes "open socket left" alerts during worker processes shutdown
> > due to pending AIO (or thread IO) operations while corresponding requests
> > have no timers. In particular, such errors were observed while reading
> > cache headers (ticket #2162), and with worker_shutdown_timeout.
> >
> > diff --git a/src/http/ngx_http_copy_filter_module.c b/src/http/ngx_http_copy_filter_module.c
> > --- a/src/http/ngx_http_copy_filter_module.c
> > +++ b/src/http/ngx_http_copy_filter_module.c
> > @@ -170,6 +170,8 @@ ngx_http_copy_aio_handler(ngx_output_cha
> > file->aio->data = r;
> > file->aio->handler = ngx_http_copy_aio_event_handler;
> >
> > + ngx_add_timer(&file->aio->event, 60000);
> > +
> > r->main->blocked++;
> > r->aio = 1;
> > ctx->aio = 1;
> > @@ -192,6 +194,17 @@ ngx_http_copy_aio_event_handler(ngx_even
> > ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
> > "http aio: \"%V?%V\"", &r->uri, &r->args);
> >
> > + if (ev->timedout) {
> > + ngx_log_error(NGX_LOG_ALERT, c->log, 0,
> > + "aio operation took too long");
> > + ev->timedout = 0;
> > + return;
> > + }
> > +
> > + if (ev->timer_set) {
> > + ngx_del_timer(ev);
> > + }
> > +
> > r->main->blocked--;
> > r->aio = 0;
> >
> > @@ -273,6 +286,8 @@ ngx_http_copy_thread_handler(ngx_thread_
> > return NGX_ERROR;
> > }
> >
> > + ngx_add_timer(&task->event, 60000);
> > +
> > r->main->blocked++;
> > r->aio = 1;
> >
> > @@ -297,6 +312,17 @@ ngx_http_copy_thread_event_handler(ngx_e
> > ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
> > "http thread: \"%V?%V\"", &r->uri, &r->args);
> >
> > + if (ev->timedout) {
> > + ngx_log_error(NGX_LOG_ALERT, c->log, 0,
> > + "thread operation took too long");
> > + ev->timedout = 0;
> > + return;
> > + }
> > +
> > + if (ev->timer_set) {
> > + ngx_del_timer(ev);
> > + }
> > +
> > r->main->blocked--;
> > r->aio = 0;
> >
> > diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c
> > --- a/src/http/ngx_http_file_cache.c
> > +++ b/src/http/ngx_http_file_cache.c
> > @@ -705,6 +705,8 @@ ngx_http_file_cache_aio_read(ngx_http_re
> > c->file.aio->data = r;
> > c->file.aio->handler = ngx_http_cache_aio_event_handler;
> >
> > + ngx_add_timer(&c->file.aio->event, 60000);
> > +
> > r->main->blocked++;
> > r->aio = 1;
> >
> > @@ -752,6 +754,17 @@ ngx_http_cache_aio_event_handler(ngx_eve
> > ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
> > "http file cache aio: \"%V?%V\"", &r->uri, &r->args);
> >
> > + if (ev->timedout) {
> > + ngx_log_error(NGX_LOG_ALERT, c->log, 0,
> > + "aio operation took too long");
> > + ev->timedout = 0;
> > + return;
> > + }
> > +
> > + if (ev->timer_set) {
> > + ngx_del_timer(ev);
> > + }
> > +
> > r->main->blocked--;
> > r->aio = 0;
> >
> > @@ -810,6 +823,8 @@ ngx_http_cache_thread_handler(ngx_thread
> > return NGX_ERROR;
> > }
> >
> > + ngx_add_timer(&task->event, 60000);
> > +
> > r->main->blocked++;
> > r->aio = 1;
> >
> > @@ -831,6 +846,17 @@ ngx_http_cache_thread_event_handler(ngx_
> > ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
> > "http file cache thread: \"%V?%V\"", &r->uri, &r->args);
> >
> > + if (ev->timedout) {
> > + ngx_log_error(NGX_LOG_ALERT, c->log, 0,
> > + "thread operation took too long");
> > + ev->timedout = 0;
> > + return;
> > + }
> > +
> > + if (ev->timer_set) {
> > + ngx_del_timer(ev);
> > + }
> > +
> > r->main->blocked--;
> > r->aio = 0;
> >
> > diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
> > --- a/src/http/ngx_http_upstream.c
> > +++ b/src/http/ngx_http_upstream.c
> > @@ -3949,6 +3949,8 @@ ngx_http_upstream_thread_handler(ngx_thr
> > r->aio = 1;
> > p->aio = 1;
> >
> > + ngx_add_timer(&task->event, 60000);
> > +
> > return NGX_OK;
> > }
> >
> > @@ -3967,6 +3969,17 @@ ngx_http_upstream_thread_event_handler(n
> > ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
> > "http upstream thread: \"%V?%V\"", &r->uri, &r->args);
> >
> > + if (ev->timedout) {
> > + ngx_log_error(NGX_LOG_ALERT, c->log, 0,
> > + "thread operation took too long");
> > + ev->timedout = 0;
> > + return;
> > + }
> > +
> > + if (ev->timer_set) {
> > + ngx_del_timer(ev);
> > + }
> > +
> > r->main->blocked--;
> > r->aio = 0;
> >
> > diff --git a/src/os/unix/ngx_files.c b/src/os/unix/ngx_files.c
> > --- a/src/os/unix/ngx_files.c
> > +++ b/src/os/unix/ngx_files.c
> > @@ -110,6 +110,8 @@ ngx_thread_read(ngx_file_t *file, u_char
> > return NGX_ERROR;
> > }
> >
> > + task->event.log = file->log;
> > +
> > file->thread_task = task;
> > }
> >
> > @@ -493,6 +495,8 @@ ngx_thread_write_chain_to_file(ngx_file_
> > return NGX_ERROR;
> > }
> >
> > + task->event.log = file->log;
> > +
> > file->thread_task = task;
> > }
> >
> > diff --git a/src/os/unix/ngx_linux_sendfile_chain.c b/src/os/unix/ngx_linux_sendfile_chain.c
> > --- a/src/os/unix/ngx_linux_sendfile_chain.c
> > +++ b/src/os/unix/ngx_linux_sendfile_chain.c
> > @@ -332,6 +332,7 @@ ngx_linux_sendfile_thread(ngx_connection
> > return NGX_ERROR;
> > }
> >
> > + task->event.log = c->log;
> > task->handler = ngx_linux_sendfile_thread_handler;
> >
> > c->sendfile_task = task;
>
> Looks good to me.
Thanks for looking, pushed to http://mdounin.ru/hg/nginx/.
--
Maxim Dounin
http://mdounin.ru/
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel
