Hello Maxim,

it appears only on two productive servers. Compiling without these modules is not possible at the moment but i've tried several times with each module deactivated (and without any of these three).

Full config is huge. I'm using a php-fpm backend with keepalive on and one upstream server with nginx and php-fpm.

http {
include mime.types;
include gzip.conf;
default_type application/octet-stream;
keepalive_timeout 240;
keepalive_requests 2000;
open_file_cache max=10000 inactive=3m;
open_file_cache_valid 3m;
open_file_cache_min_uses 2;
open_file_cache_errors off;
index wartung.html index.php index.html index.htm;
recursive_error_pages on;

large_client_header_buffers 8 256k;
client_max_body_size 1024m;
client_body_buffer_size 128K;
client_header_buffer_size 10k;
client_header_timeout 120;
client_body_timeout 120;
fastcgi_read_timeout 240;

set_real_ip_from x.x.x.0/24;
real_ip_header X-Forwarded-For;

#nginx 1.0
#limit_zone spiders $binary_remote_addr 10m;
#nginx 1.1
limit_conn_zone $binary_remote_addr zone=spiders:10m;

limit_req_zone $binary_remote_addr zone=spider_req:10m rate=5r/s;
limit_conn_log_level info;

upstream be_server_nginx {
server IP:80;
keepalive 300;
}
}

...

location @be_nginx {
proxy_pass http://be_server_nginx;
proxy_set_header Connection ""; proxy_http_version 1.1;
include /etc/nginx/proxy.conf;


output of gdb:

(gdb) fr 0
#0 0x000000000042cbe8 in ngx_readv_chain (c=0x2b508a1d43d0, chain=0x602348) at src/os/unix/ngx_readv_chain.c:195
195 in src/os/unix/ngx_readv_chain.c

(gdb) p chain
$1 = (ngx_chain_t *) 0x602348

(gdb) p *chain
$2 = {buf = 0x602460, next = 0x752628}

(gdb) fr 1
#1 0x000000000042b133 in ngx_event_pipe (p=0x747170, do_write=<value optimized out>) at src/event/ngx_event_pipe.c:271
271 src/event/ngx_event_pipe.c: No such file or directory.
in src/event/ngx_event_pipe.c

(gdb) p *p
$3 = {upstream = 0x2b508a1d43d0, downstream = 0x2b508a1d4550, free_raw_bufs = 0x0, in = 0x6024d0, last_in = 0x7523a0, out = 0x0, free = 0x602450, busy = 0x0,
input_filter = 0x47bc30 <ngx_http_fastcgi_input_filter>, input_ctx = 0x69ef00, output_filter = 0x43ca40 <ngx_http_output_filter>, output_ctx = 0x69ef00, read = 0, cacheable = 0,
single_buf = 0, free_bufs = 1, upstream_done = 0, upstream_error = 0, upstream_eof = 0, upstream_blocked = 0, downstream_done = 0, downstream_error = 0, cyclic_temp_file = 0, allocated = 2,
bufs = {num = 10, size = 262144}, tag = 0x5c8480, busy_size = 262144, read_length = 378731, length = 1, max_temp_file_size = 0, temp_file_write_size = 524288, read_timeout = 180000,
send_timeout = 60000, send_lowat = 0, pool = 0x69b6e0, log = 0x6c5380, preread_bufs = 0x0, preread_size = 129547, buf_to_file = 0x0, temp_file = 0x602358, num = 6}

(gdb) fr 2
#2 0x000000000045410e in ngx_http_upstream_process_upstream (r=0x69ef00, u=0x746db8) at src/http/ngx_http_upstream.c:2641
2641 src/http/ngx_http_upstream.c: No such file or directory.
in src/http/ngx_http_upstream.c

(gdb) p *r
$4 = {signature = 1347703880, connection = 0x2b508a1d4550, ctx = 0x69baf0, main_conf = 0x5f7910, srv_conf = 0x617f48, loc_conf = 0x640e00,
read_event_handler = 0x454510 <ngx_http_upstream_rd_check_broken_connection>, write_event_handler = 0x454b50 <ngx_http_upstream_process_downstream>, cache = 0x0, upstream = 0x746db8,
upstream_states = 0x747890, pool = 0x69b6e0, header_in = 0x6c5450, headers_in = {headers = {last = 0x69ef70, part = {elts = 0x69c008, nelts = 11, next = 0x0}, size = 48, nalloc = 20,
pool = 0x69b6e0}, host = 0x69c0f8, connection = 0x0, if_modified_since = 0x0, if_unmodified_since = 0x0, user_agent = 0x69c098, referer = 0x69c038, content_length = 0x0,
content_type = 0x0, range = 0x0, if_range = 0x0, transfer_encoding = 0x0, expect = 0x0, accept_encoding = 0x69c0c8, via = 0x69c188, authorization = 0x0, keep_alive = 0x0,
x_forwarded_for = 0x69c1b8, x_real_ip = 0x0, depth = 0x0, destination = 0x0, overwrite = 0x0, date = 0x0, user = {len = 0, data = 0x0}, passwd = {len = 0, data = 0x0}, cookies = {
elts = 0x69c3c8, nelts = 1, size = 8, nalloc = 2, pool = 0x69b6e0}, server = {len = 22, data = 0x74eadb "domain.com"}, content_length_n = -1, keep_alive_n = -1,
connection_type = 2, msie = 1, msie6 = 0, opera = 0, gecko = 0, chrome = 0, safari = 0, konqueror = 0}, headers_out = {headers = {last = 0x69f0c8, part = {elts = 0x69b730, nelts = 5,
next = 0x0}, size = 48, nalloc = 20, pool = 0x69b6e0}, status = 200, status_line = {len = 6, data = 0x4a2662 "200 OK"}, server = 0x69b790, date = 0x0, content_length = 0x0,
content_encoding = 0x69b7f0, location = 0x0, refresh = 0x0, last_modified = 0x0, content_range = 0x0, accept_ranges = 0x0, www_authenticate = 0x0, expires = 0x0, etag = 0x0,
override_charset = 0x0, content_type_len = 9, content_type = {len = 24, data = 0x747a0d "text/html; charset=utf-8"}, charset = {len = 5, data = 0x747a20 "utf-8"},
content_type_lowcase = 0x602078 "text/html", content_type_hash = 101824002410317, cache_control = {elts = 0x6016c8, nelts = 1, size = 8, nalloc = 2, pool = 0x69b6e0}, content_length_n = -1,
date_time = 0, last_modified_time = -1}, request_body = 0x747270, lingering_time = 0, start_sec = 1334650957, start_msec = 91, method = 2, http_version = 1001, request_line = {len = 50,
data = 0x74e8c0 "GET /nc/pdf-generation.html?pageID=776404 HTTP/1.1\r\nAccept"}, uri = {len = 10, data = 0x746d62 "/index.php"}, args = {len = 13, data = 0x746a8f "pageID=776404"}, exten = {
len = 3, data = 0x746d69 "php"}, unparsed_uri = {len = 37, data = 0x74e8c4 "/nc/pdf-generation.html?pageID=776404 HTTP/1.1\r\nAccept"}, method_name = {len = 3,
data = 0x74e8c0 "GET /nc/pdf-generation.html?pageID=776404 HTTP/1.1\r\nAccept"}, http_protocol = {len = 8, data = 0x74e8ea "HTTP/1.1\r\nAccept"}, out = 0x7523b8, main = 0x69ef00,
parent = 0x0, postponed = 0x0, post_subrequest = 0x0, posted_requests = 0x0, virtual_names = 0x744098, phase_handler = 13, content_handler = 0x47b940 <ngx_http_fastcgi_handler>,
access_code = 0, variables = 0x69bc98, ncaptures = 0, captures = 0x69c580, captures_data = 0x746a84 "/index.php?pageID=776404", limit_rate = 0, header_size = 339, request_length = 785,
err_status = 0, http_connection = 0x6c53d8, log_handler = 0x442e60 <ngx_http_log_error_handler>, cleanup = 0x7478f0, subrequests = 201, count = 1, blocked = 0, aio = 0, http_state = 2,
complex_uri = 0, quoted_uri = 0, plus_in_uri = 0, space_in_uri = 0, invalid_header = 0, add_uri_to_alias = 0, valid_location = 1, valid_unparsed_uri = 0, uri_changed = 0, uri_changes = 10,
request_body_in_single_buf = 0, request_body_in_file_only = 0, request_body_in_persistent_file = 0, request_body_in_clean_file = 0, request_body_file_group_access = 0,
request_body_file_log_level = 5, subrequest_in_memory = 0, waited = 0, cached = 0, gzip_tested = 1, gzip_ok = 1, gzip_vary = 1, proxy = 0, bypass_cache = 0, no_cache = 0, limit_conn_set = 1,
limit_req_set = 1, pipeline = 0, plain_http = 0, chunked = 1, header_only = 0, keepalive = 1, lingering_close = 0, discard_body = 0, internal = 1, error_page = 0, ignore_content_encoding = 0,
filter_finalize = 0, post_action = 0, request_complete = 0, request_output = 1, header_sent = 1, expect_tested = 0, root_tested = 0, done = 0, logged = 0, buffered = 0,
main_filter_need_in_memory = 1, filter_need_in_memory = 1, filter_need_temporary = 0, allow_ranges = 0, stat_reading = 0, stat_writing = 1, state = 0, header_hash = 6740241951503570450,
lowcase_index = 13, lowcase_header = "cache-controlsitiont", '\0' <repeats 11 times>,
header_name_start = 0x2b50cc8430b3 "^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×ÿÐßãߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^"...,
header_name_end = 0x2b50cc8430a8 "û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×ÿÐßãߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û"...,
header_start = 0x2b50cc8430aa "uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×ÿÐßãߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯u"...,
header_end = 0x2b50cc8430b3 "^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×ÿÐßãߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^"..., uri_start = 0x74e8c4 "/nc/pdf-generation.html?pageID=776404 HTTP/1.1\r\nAccept", uri_end = 0x74e8e9 " HTTP/1.1\r\nAccept",
uri_ext = 0x74e8d7 "html?pageID=776404 HTTP/1.1\r\nAccept", args_start = 0x74e8dc "pageID=776404 HTTP/1.1\r\nAccept",
request_start = 0x74e8c0 "GET /nc/pdf-generation.html?pageID=776404 HTTP/1.1\r\nAccept", request_end = 0x74e8f2 "\r\nAccept",
method_end = 0x74e8c2 "T /nc/pdf-generation.html?pageID=776404 HTTP/1.1\r\nAccept", schema_start = 0x0, schema_end = 0x0, host_start = 0x0, host_end = 0x0, port_start = 0x0, port_end = 0x0,
http_minor = 1, http_major = 1}

(gdb) p *u
$5 = {read_event_handler = 0x454090 <ngx_http_upstream_process_upstream>, write_event_handler = 0x452980 <ngx_http_upstream_dummy_handler>, peer = {connection = 0x2b508a1d43d0,
sockaddr = 0x617ea8, socklen = 110, name = 0x68a6b0, tries = 1, get = 0x487370 <ngx_http_upstream_get_keepalive_peer>, free = 0x4875b0 <ngx_http_upstream_free_keepalive_peer>,
data = 0x747908, set_session = 0x486e80 <ngx_http_upstream_keepalive_set_session>, save_session = 0x486e90 <ngx_http_upstream_keepalive_save_session>, local = 0x0, rcvbuf = 0,
log = 0x6c5380, cached = 1, log_error = 1}, pipe = 0x747170, request_bufs = 0x747880, output = {buf = 0x0, in = 0x0, free = 0x0, busy = 0x0, sendfile = 0, directio = 0, unaligned = 0,
need_in_memory = 0, need_in_temp = 0, alignment = 512, pool = 0x69b6e0, allocated = 0, bufs = {num = 1, size = 131072}, tag = 0x5c8480, output_filter = 0x414db0 <ngx_chain_writer>,
filter_ctx = 0x746eb0}, writer = {out = 0x0, last = 0x746eb0, connection = 0x2b508a1d43d0, pool = 0x69b6e0, limit = 0}, conf = 0x6416a8, headers_in = {headers = {last = 0x746ee8, part = {
elts = 0x601520, nelts = 4, next = 0x0}, size = 48, nalloc = 8, pool = 0x69b6e0}, status_n = 200, status_line = {len = 6, data = 0x4a2662 "200 OK"}, status = 0x0, date = 0x0,
server = 0x0, connection = 0x0, expires = 0x0, etag = 0x0, x_accel_expires = 0x0, x_accel_redirect = 0x0, x_accel_limit_rate = 0x0, content_type = 0x601580, content_length = 0x601520,
last_modified = 0x0, location = 0x0, accept_ranges = 0x0, www_authenticate = 0x0, transfer_encoding = 0x0, content_encoding = 0x0, content_length_n = 355230, cache_control = {
elts = 0x747a38, nelts = 1, size = 8, nalloc = 2, pool = 0x69b6e0}, connection_close = 0, chunked = 0}, resolved = 0x0, buffer = {
pos = 0x2b50cc843010 "¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î"...,
last = 0x2b50cc843010 "¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î"..., file_pos = 0, file_last = 0,
start = 0x2b50cc843010 "¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î½×½û¯uï~ëÝ{ߺ÷^÷î"..., end = 0x2b50cc883010 "", tag = 0x5c8480, file = 0x0, shadow = 0x0, temporary = 1, memory = 0, mmap = 0, recycled = 1, in_file = 0, flush = 0, sync = 0,
last_buf = 0, last_in_chain = 0, last_shadow = 0, temp_file = 0, num = 2}, length = 355230, out_bufs = 0x0, busy_bufs = 0x0, free_bufs = 0x0,
input_filter_init = 0x47af80 <ngx_http_fastcgi_input_filter_init>, input_filter = 0, input_filter_ctx = 0x0, create_key = 0x47c100 <ngx_http_fastcgi_create_key>,
create_request = 0x47c3a0 <ngx_http_fastcgi_create_request>, reinit_request = 0x47af50 <ngx_http_fastcgi_reinit_request>, process_header = 0x47cf20 <ngx_http_fastcgi_process_header>,
abort_request = 0x47b560 <ngx_http_fastcgi_abort_request>, finalize_request = 0x47b530 <ngx_http_fastcgi_finalize_request>, rewrite_redirect = 0, rewrite_cookie = 0, timeout = 0,
state = 0x7478b8, method = {len = 0, data = 0x0}, schema = {len = 10, data = 0x4a8c69 "fastcgi://"}, uri = {len = 0, data = 0x0}, cleanup = 0x7478f0, store = 0, cacheable = 0, accel = 0,
ssl = 0, cache_status = 0, buffering = 1, keepalive = 0, request_sent = 1, header_sent = 1}