Dear all,
error logs show this error (worker process exited on signal 11). The error appears after ~1 hour, then with the half of the worker processes. No problems with 1.0-versions of nginx. Problem existed in 1.1.18 already.
I'm using a custom rpm build with the following additional modules:
--add-module=/root/rpm/SOURCES/agentzh-headers-more-nginx-module-3580526 \
--add-module=/root/rpm/SOURCES/FRiCKLE-ngx_cache_purge-e34bd2c \
--add-module=/root/rpm/SOURCES/substitutions4nginx-read-only \
Here is the backtrace, any ideas?
Thank you.
2012/04/17 10:16:36 [alert] 20802#0: worker process 20808 exited on signal 11 (core dumped)
2012/04/17 10:16:45 [alert] 20802#0: worker process 20805 exited on signal 11 (core dumped)
2012/04/17 10:17:33 [alert] 20802#0: worker process 23616 exited on signal 11 (core dumped)
2012/04/17 10:18:14 [alert] 20802#0: worker process 20806 exited on signal 11 (core dumped)
2012/04/17 10:22:41 [alert] 20802#0: worker process 23892 exited on signal 11 (core dumped)
dmesg:
nginx[20808]: segfault at 0000000000000020 rip 000000000042a47b rsp 00007fff79073f80 error 4
nginx[20805]: segfault at 0000000000000020 rip 000000000042a47b rsp 00007fff79073f80 error 4
nginx[23616]: segfault at 0000000000000020 rip 000000000042a47b rsp 00007fff79073ff0 error 4
nginx[20806]: segfault at 0000000000000020 rip 000000000042a47b rsp 00007fff79073f80 error 4
nginx[23892]: segfault at 0000000000000008 rip 000000000042cbe8 rsp 00007fff79073f20 error 6
gdb /usr/sbin/nginx /var/local/dumps/core
GNU gdb 6.6
Copyright (C) 2006 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "x86_64-suse-linux"...
Using host libthread_db library "/lib64/libthread_db.so.1".
Reading symbols from /lib64/libpthread.so.0...done.
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libcrypt.so.1...done.
Loaded symbols for /lib64/libcrypt.so.1
Reading symbols from /usr/lib64/libpcre.so.0...done.
Loaded symbols for /usr/lib64/libpcre.so.0
Reading symbols from /usr/lib64/libssl.so.0.9.8...done.
Loaded symbols for /usr/lib64/libssl.so.0.9.8
Reading symbols from /usr/lib64/libcrypto.so.0.9.8...done.
Loaded symbols for /usr/lib64/libcrypto.so.0.9.8
Reading symbols from /lib64/libdl.so.2...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libz.so.1...done.
Loaded symbols for /lib64/libz.so.1
Reading symbols from /usr/lib64/libGeoIP.so.1...done.
Loaded symbols for /usr/lib64/libGeoIP.so.1
Reading symbols from /usr/lib/perl5/5.8.8/x86_64-linux-thread-multi/CORE/libperl.so...done.
Loaded symbols for /usr/lib/perl5/5.8.8/x86_64-linux-thread-multi/CORE/libperl.so
Reading symbols from /lib64/libm.so.6...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /lib64/libc.so.6...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/lib/perl5/site_perl/5.8.8/x86_64-linux-thread-multi/auto/nginx/nginx.so...done.
Loaded symbols for /usr/lib/perl5/site_perl/5.8.8/x86_64-linux-thread-multi/auto/nginx/nginx.so
Core was generated by `nginx: worker pr'.
Program terminated with signal 11, Segmentation fault.
#0 0x000000000042cbe8 in ngx_readv_chain (c=0x2b508a1d43d0, chain=0x602348) at src/os/unix/ngx_readv_chain.c:195
195 src/os/unix/ngx_readv_chain.c: No such file or directory.
in src/os/unix/ngx_readv_chain.c
(gdb) bt
#0 0x000000000042cbe8 in ngx_readv_chain (c=0x2b508a1d43d0, chain=0x602348) at src/os/unix/ngx_readv_chain.c:195
#1 0x000000000042b133 in ngx_event_pipe (p=0x747170, do_write=<value optimized out>) at src/event/ngx_event_pipe.c:271
#2 0x000000000045410e in ngx_http_upstream_process_upstream (r=0x69ef00, u=0x746db8) at src/http/ngx_http_upstream.c:2641
#3 0x0000000000454269 in ngx_http_upstream_handler (ev=0x2b50cc7d9218) at src/http/ngx_http_upstream.c:935
#4 0x0000000000428f64 in ngx_event_process_posted (cycle=0x5f6490, posted=0x5d14a0) at src/event/ngx_event_posted.c:40
#5 0x000000000042fc36 in ngx_worker_process_cycle (cycle=0x5f6490, data=<value optimized out>) at src/os/unix/ngx_process_cycle.c:806
#6 0x000000000042e14d in ngx_spawn_process (cycle=0x5f6490, proc=0x42fb70 <ngx_worker_process_cycle>, data=0x0, name=0x49e05e "worker process", respawn=5) at src/os/unix/ngx_process.c:198
#7 0x0000000000430c76 in ngx_master_process_cycle (cycle=0x5f6490) at src/os/unix/ngx_process_cycle.c:622
#8 0x000000000041235d in main (argc=1, argv=<value optimized out>) at src/core/nginx.c:410
(gdb) backtrace full
#0 0x000000000042cbe8 in ngx_readv_chain (c=0x2b508a1d43d0, chain=0x602348) at src/os/unix/ngx_readv_chain.c:195
prev = (u_char *) 0x0
n = <value optimized out>
size = 0
err = <value optimized out>
vec = {elts = 0x7fff79073f20, nelts = 0, size = 16, nalloc = 16, pool = 0x5f57a0}
rev = <value optimized out>
iov = (struct iovec *) 0x0
iovs = {{iov_base = 0x40000, iov_len = 140735223906344}, {iov_base = 0x69ef00, iov_len = 47624914552144}, {iov_base = 0x6024c0, iov_len = 262144}, {iov_base = 0x1, iov_len = 4566241}, {
iov_base = 0x40000, iov_len = 6942464}, {iov_base = 0x2b508a1d4550, iov_len = 0}, {iov_base = 0x0, iov_len = 47624930312022}, {iov_base = 0x69b6e0, iov_len = 6942464}, {
iov_base = 0x2b508a1d4550, iov_len = 6300864}, {iov_base = 0x40000, iov_len = 1}, {iov_base = 0x602460, iov_len = 4367464}, {iov_base = 0x1, iov_len = 2}, {iov_base = 0x0,
iov_len = 47624930399933}, {iov_base = 0x40000, iov_len = 4309842}, {iov_base = 0x2b50cc80e2e8, iov_len = 4370439}, {iov_base = 0x2b, iov_len = 2}}
#1 0x000000000042b133 in ngx_event_pipe (p=0x747170, do_write=<value optimized out>) at src/event/ngx_event_pipe.c:271
rc = <value optimized out>
rev = <value optimized out>
wev = <value optimized out>
#2 0x000000000045410e in ngx_http_upstream_process_upstream (r=0x69ef00, u=0x746db8) at src/http/ngx_http_upstream.c:2641
c = (ngx_connection_t *) 0x2b508a1d43d0
#3 0x0000000000454269 in ngx_http_upstream_handler (ev=0x2b50cc7d9218) at src/http/ngx_http_upstream.c:935
c = (ngx_connection_t *) 0x2b508a1d4550
r = (ngx_http_request_t *) 0x69ef00
u = (ngx_http_upstream_t *) 0x746db8
#4 0x0000000000428f64 in ngx_event_process_posted (cycle=0x5f6490, posted=0x5d14a0) at src/event/ngx_event_posted.c:40
ev = (ngx_event_t *) 0x2b50cc7d9218
#5 0x000000000042fc36 in ngx_worker_process_cycle (cycle=0x5f6490, data=<value optimized out>) at src/os/unix/ngx_process_cycle.c:806
i = 0
c = <value optimized out>
#6 0x000000000042e14d in ngx_spawn_process (cycle=0x5f6490, proc=0x42fb70 <ngx_worker_process_cycle>, data=0x0, name=0x49e05e "worker process", respawn=5) at src/os/unix/ngx_process.c:198
on = 1
pid = 0
s = 5
#7 0x0000000000430c76 in ngx_master_process_cycle (cycle=0x5f6490) at src/os/unix/ngx_process_cycle.c:622
title = <value optimized out>
p = <value optimized out>
size = 9
i = <value optimized out>
n = 432
sigio = 0
set = {__val = {0 <repeats 16 times>}}
itv = {it_interval = {tv_sec = 0, tv_usec = 6251688}, it_value = {tv_sec = 0, tv_usec = 47624916165168}}
live = 1
delay = 0
ccf = (ngx_core_conf_t *) 0x5f74f0
#8 0x000000000041235d in main (argc=1, argv=<value optimized out>) at src/core/nginx.c:410
i = <value optimized out>
log = (ngx_log_t *) 0x5ce6e0
cycle = (ngx_cycle_t *) 0x5f6490
init_cycle = {conf_ctx = 0x0, pool = 0x5f5440, log = 0x5ce6e0, new_log = {log_level = 0, file = 0x0, connection = 0, handler = 0, data = 0x0, action = 0x0}, files = 0x0,
free_connections = 0x0, free_connection_n = 0, reusable_connections_queue = {prev = 0x0, next = 0x0}, listening = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, pathes = {
elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, open_files = {last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0}, shared_memory = {
last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0}, connection_n = 0, files_n = 0, connections = 0x0, read_events = 0x0, write_events = 0x0,
old_cycle = 0x0, conf_file = {len = 21, data = 0x4998b3 "/etc/nginx/nginx.conf"}, conf_param = {len = 0, data = 0x0}, conf_prefix = {len = 11, data = 0x4998b3 "/etc/nginx/nginx.conf"},
prefix = {len = 6, data = 0x4998ac "/usr//"}, lock_file = {len = 0, data = 0x0}, hostname = {len = 0, data = 0x0}}
ccf = (ngx_core_conf_t *) 0x5f74f0
