Hi expert,

I am running into nginx core dump issue and I need experts help in debugging. Thank you so much!

we are running nginx 1.6.1 on centos 5.6

nginx version: nginx/1.6.1
built by gcc 4.1.2 20080704 (Red Hat 4.1.2-54)
TLS SNI support enabled
configure arguments: --prefix=/Local/ThirdParty/nginx --sbin-path=/Local/ThirdParty/nginx/nginx --conf-path=/Local/ThirdParty/nginx/conf/AOS-nginx.conf --error-log-path=/Local/ThirdParty/nginx/logs/AOS-error.log --with-http_stub_status_module --pid-path=/Local/ThirdParty/nginx/runtime/nginx.pid --lock-path=/Local/ThirdParty/nginx/runtime/nginx.lock --with-http_ssl_module --with-http_gzip_static_module --with-pcre=/home/worun/pcre-8.35 --with-openssl=/home/worun/openssl-0.9.8y --with-http_realip_module --with-debug --add-module=/home/worun/nginx-auth-ldap-0.1/

out put from the gdb core dump
GNU gdb (GDB) Red Hat Enterprise Linux (7.0.1-45.0.1.el5)
Copyright (C) 2009 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later http://gnu.org/licenses/gpl.html
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
http://www.gnu.org/software/gdb/bugs/...
Reading symbols from /Local/ThirdParty/nginx/nginx...done.
[New Thread 28390]
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[Thread debugging using libthread_db enabled]
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libcrypt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcrypt.so.1
Reading symbols from /usr/lib64/libldap-2.3.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libldap-2.3.so.0
Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /usr/lib64/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libz.so.1
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/lib64/liblber-2.3.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/liblber-2.3.so.0
Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libresolv.so.2
Reading symbols from /usr/lib64/libsasl2.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libsasl2.so.2
Reading symbols from /lib64/libssl.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libssl.so.6
Reading symbols from /lib64/libcrypto.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcrypto.so.6
Reading symbols from /usr/lib64/libgssapi_krb5.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libgssapi_krb5.so.2
Reading symbols from /usr/lib64/libkrb5.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libkrb5.so.3
Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcom_err.so.2
Reading symbols from /usr/lib64/libk5crypto.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libk5crypto.so.3
Reading symbols from /usr/lib64/libkrb5support.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libkrb5support.so.0
Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkeyutils.so.1
Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libselinux.so.1
Reading symbols from /lib64/libsepol.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libsepol.so.1
Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgcc_s.so.1

warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fffe65fc000
Core was generated by `nginx: w'.
Program terminated with signal 6, Aborted.
#0 0x00002adaf5619265 in raise () from /lib64/libc.so.6
(gdb) bt
#0 0x00002adaf5619265 in raise () from /lib64/libc.so.6
#1 0x00002adaf561ad10 in abort () from /lib64/libc.so.6
#2 0x00002adaf565399b in __libc_message () from /lib64/libc.so.6
#3 0x00002adaf565b45f in _int_free () from /lib64/libc.so.6
#4 0x00002adaf565b8bb in free () from /lib64/libc.so.6
#5 0x000000000042bbe0 in ngx_destroy_pool (pool=0x8f6f010) at src/core/ngx_palloc.c:87
#6 0x000000000047de82 in ngx_http_free_request (r=0x8f6f060, rc=0) at src/http/ngx_http_request.c:3492
#7 0x000000000047cae2 in ngx_http_set_keepalive (r=0x8f6f060) at src/http/ngx_http_request.c:2894
#8 0x000000000047c091 in ngx_http_finalize_connection (r=0x8f6f060) at src/http/ngx_http_request.c:2537
#9 0x000000000047bd38 in ngx_http_finalize_request (r=0x8f6f060, rc=0) at src/http/ngx_http_request.c:2438
#10 0x00000000004992e4 in ngx_http_upstream_finalize_request (r=0x8f6f060, u=0x8fd6a50, rc=0) at src/http/ngx_http_upstream.c:3558
#11 0x00000000004984b3 in ngx_http_upstream_process_request (r=0x8f6f060) at src/http/ngx_http_upstream.c:3159
#12 0x00000000004981f8 in ngx_http_upstream_process_upstream (r=0x8f6f060, u=0x8fd6a50) at src/http/ngx_http_upstream.c:3090
#13 0x0000000000496c86 in ngx_http_upstream_send_response (r=0x8f6f060, u=0x8fd6a50) at src/http/ngx_http_upstream.c:2493
#14 0x0000000000495168 in ngx_http_upstream_process_header (r=0x8f6f060, u=0x8fd6a50) at src/http/ngx_http_upstream.c:1735
#15 0x0000000000493a32 in ngx_http_upstream_handler (ev=0x8fa0c70) at src/http/ngx_http_upstream.c:977
#16 0x00000000004502b4 in ngx_event_process_posted (cycle=0x8f64030, posted=0x872da8) at src/event/ngx_event_posted.c:40
#17 0x000000000044ddd5 in ngx_process_events_and_timers (cycle=0x8f64030) at src/event/ngx_event.c:275
#18 0x000000000045b080 in ngx_worker_process_cycle (cycle=0x8f64030, data=0x0) at src/os/unix/ngx_process_cycle.c:816
#19 0x0000000000457b8d in ngx_spawn_process (cycle=0x8f64030, proc=0x45ae92 <ngx_worker_process_cycle>, data=0x0, name=0x5cb0f6 "worker process", respawn=0)
at src/os/unix/ngx_process.c:198
#20 0x000000000045aaab in ngx_reap_children (cycle=0x8f64030) at src/os/unix/ngx_process_cycle.c:627
#21 0x00000000004597dc in ngx_master_process_cycle (cycle=0x8f64030) at src/os/unix/ngx_process_cycle.c:180
#22 0x0000000000428904 in main (argc=1, argv=0x7fffe6534ef8) at src/core/nginx.c:407
(gdb)

(gdb) bt full
#0 0x00002adaf5619265 in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x00002adaf561ad10 in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x00002adaf565399b in __libc_message () from /lib64/libc.so.6
No symbol table info available.
#3 0x00002adaf565b45f in _int_free () from /lib64/libc.so.6
No symbol table info available.
#4 0x00002adaf565b8bb in free () from /lib64/libc.so.6
No symbol table info available.
#5 0x000000000042bbe0 in ngx_destroy_pool (pool=0x8f6f010) at src/core/ngx_palloc.c:87
p = 0x8f6f010
n = 0x8fd6a30
l = 0x0
c = 0x0
#6 0x000000000047de82 in ngx_http_free_request (r=0x8f6f060, rc=0) at src/http/ngx_http_request.c:3492
log = 0x8f790b0
pool = 0x8f6f010
linger = {l_onoff = -430750128, l_linger = 32767}
cln = 0x0
ctx = 0x8f79180
clcf = 0x42c1b4
#7 0x000000000047cae2 in ngx_http_set_keepalive (r=0x8f6f060) at src/http/ngx_http_request.c:2894
tcp_nodelay = 10970
i = 150828568
b = 0x8f79208
f = 0x8fd7618
rev = 0x8fa0c08
wev = 0x4a27c9
c = 0x2adaf4979448
hc = 0x8f790f8
cscf = 0x7fffe65346f0
clcf = 0x8f93790
#8 0x000000000047c091 in ngx_http_finalize_connection (r=0x8f6f060) at src/http/ngx_http_request.c:2537
clcf = 0x8f93790
#9 0x000000000047bd38 in ngx_http_finalize_request (r=0x8f6f060, rc=0) at src/http/ngx_http_request.c:2438
c = 0x2adaf4979448
pr = 0x0
clcf = 0x8fd7680
#10 0x00000000004992e4 in ngx_http_upstream_finalize_request (r=0x8f6f060, u=0x8fd6a50, rc=0) at src/http/ngx_http_upstream.c:3558
flush = 0
tp = 0x86c330
#11 0x00000000004984b3 in ngx_http_upstream_process_request (r=0x8f6f060) at src/http/ngx_http_upstream.c:3159
tf = 0x100000010
p = 0x8fd6e98
u = 0x8fd6a50
#12 0x00000000004981f8 in ngx_http_upstream_process_upstream (r=0x8f6f060, u=0x8fd6a50) at src/http/ngx_http_upstream.c:3090
c = 0x2adaf4979520
#13 0x0000000000496c86 in ngx_http_upstream_send_response (r=0x8f6f060, u=0x8fd6a50) at src/http/ngx_http_upstream.c:2493
---Type <return> to continue, or q <return> to quit---
tcp_nodelay = 0
n = 140737057605712
rc = 0
p = 0x8fd6e98
c = 0x2adaf4979448
clcf = 0x8f93790
#14 0x0000000000495168 in ngx_http_upstream_process_header (r=0x8f6f060, u=0x8fd6a50) at src/http/ngx_http_upstream.c:1735
n = 842
rc = 0
c = 0x2adaf4979520
#15 0x0000000000493a32 in ngx_http_upstream_handler (ev=0x8fa0c70) at src/http/ngx_http_upstream.c:977
c = 0x2adaf4979448
r = 0x8f6f060
ctx = 0x8f79180
u = 0x8fd6a50
#16 0x00000000004502b4 in ngx_event_process_posted (cycle=0x8f64030, posted=0x872da8) at src/event/ngx_event_posted.c:40
ev = 0x8fa0c70
#17 0x000000000044ddd5 in ngx_process_events_and_timers (cycle=0x8f64030) at src/event/ngx_event.c:275
flags = 3
timer = 59971
delta = 28
#18 0x000000000045b080 in ngx_worker_process_cycle (cycle=0x8f64030, data=0x0) at src/os/unix/ngx_process_cycle.c:816
worker = 0
i = 140737057607408
c = 0x0
#19 0x0000000000457b8d in ngx_spawn_process (cycle=0x8f64030, proc=0x45ae92 <ngx_worker_process_cycle>, data=0x0, name=0x5cb0f6 "worker process", respawn=0)
at src/os/unix/ngx_process.c:198
on = 1
pid = 0
s = 0
#20 0x000000000045aaab in ngx_reap_children (cycle=0x8f64030) at src/os/unix/ngx_process_cycle.c:627
i = 0
n = 8
live = 0
ch = {command = 2, pid = 28371, slot = 0, fd = -1}
ccf = 0x53ee7696
#21 0x00000000004597dc in ngx_master_process_cycle (cycle=0x8f64030) at src/os/unix/ngx_process_cycle.c:180
title = 0x8f9dbb4 "master process ./nginx"
p = 0x8f9dbca ""
size = 23
i = 1
n = 0
sigio = 0
set = {__val = {0 <repeats 16 times>}}
itv = {it_interval = {tv_sec = 0, tv_usec = 8644424}, it_value = {tv_sec = 8, tv_usec = 41}}
live = 1
delay = 0
ls = 0x0
ccf = 0x8f65000
---Type <return> to continue, or q <return> to quit---
#22 0x0000000000428904 in main (argc=1, argv=0x7fffe6534ef8) at src/core/nginx.c:407
i = 52
log = 0x86c180
cycle = 0x8f64030
init_cycle = {conf_ctx = 0x0, pool = 0x8f63690, log = 0x86c180, new_log = {log_level = 0, file = 0x0, connection = 0, handler = 0, data = 0x0,
action = 0x0, next = 0x0}, log_use_stderr = 0, files = 0x0, free_connections = 0x0, free_connection_n = 0, reusable_connections_queue = {prev = 0x0,
next = 0x0}, listening = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, paths = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0},
open_files = {last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0}, shared_memory = {last = 0x0, part = {elts = 0x0,
nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0}, connection_n = 0, files_n = 0, connections = 0x0, read_events = 0x0, write_events = 0x0,
old_cycle = 0x0, conf_file = {len = 43, data = 0x5c7378 "/Local/ThirdParty/nginx/conf/AOS-nginx.conf"}, conf_param = {len = 0, data = 0x0},
conf_prefix = {len = 29, data = 0x5c7378 "/Local/ThirdParty/nginx/conf/AOS-nginx.conf"}, prefix = {len = 24, data = 0x5c735e "/Local/ThirdParty/nginx/"},
lock_file = {len = 0, data = 0x0}, hostname = {len = 0, data = 0x0}}
ccf = 0x8f65000
(gdb)