Hello

our nginx version is nginx-1.9.7

nginx.conf include:
worker_processes auto;
output_buffers 1 128k;
sendfile on;
tcp_nopush on;
sendfile_max_chunk 128k;
keepalive_timeout 8;
server_tokens off;
client_max_body_size 10M;
......

all nginx worker include cache manager is D state after startup 7 days, more importantly, the situation has appeared on other three servers

nginx is used to as cdn node , and load is not very high

does the reason is nginx itself or xfs filesystem or others？

any response will be appreciated.

analysis：
it look like all nginx worker blocked on a file that one worker write but others read , because we found that file "/data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7" was accessed by 11 workers, and one of ps FD is 50u
but we can't explain this situation , please help.
------------------------------------------------------------------------------------------------------------------------------------------------------------------
[root@w-fc12 conf]# for i in `ps aux | awk '$NF=="process" {print $2}'` ; do lsof -p $i|grep /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7; done
nginx 15269 nobody 270r REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7
nginx 15269 nobody 760r REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7
nginx 15273 nobody 50r REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7
nginx 15274 nobody 931r REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7
nginx 15277 nobody 622r REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7
nginx 15285 nobody 50u REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7
nginx 15291 nobody 131r REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7
nginx 15291 nobody 714r REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7
nginx 15294 nobody 56r REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7
nginx 15294 nobody 400r REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7
nginx 15307 nobody 594r REG 251,0 41144 268717925 /data01/nginx_proxy_cache/cache/e7/c2/aa3d638afdddd44220bdf1954f5dc2e7

--------------------------------------------------------------------------------------------------------------------------------------------------------------------
[liuguirong@w-fc12 conf]$ ps axu|grep nginx
root 4879 0.0 0.1 9687712 89016 ? Ds Aug04 0:00 nginx: master process /usr/local/nginx/sbin/nginx
nobody 15268 1.3 2.2 9740068 1454828 ? D Aug05 123:55 nginx: worker process
nobody 15269 1.3 2.2 9736804 1451504 ? D Aug05 123:06 nginx: worker process
nobody 15270 1.3 2.2 9738568 1453272 ? D Aug05 123:17 nginx: worker process
nobody 15271 1.3 2.2 9739520 1454184 ? D Aug05 123:14 nginx: worker process
nobody 15272 1.3 2.2 9740440 1455216 ? D Aug05 124:39 nginx: worker process
nobody 15273 1.3 2.2 9742540 1457160 ? D Aug05 123:52 nginx: worker process
nobody 15274 1.3 2.2 9737832 1452544 ? D Aug05 123:43 nginx: worker process
nobody 15275 1.3 2.2 9741376 1456092 ? D Aug05 122:14 nginx: worker process
nobody 15276 1.3 2.2 9738660 1453544 ? D Aug05 122:46 nginx: worker process
nobody 15277 1.3 2.2 9740412 1455124 ? D Aug05 123:37 nginx: worker process
nobody 15278 1.3 2.2 9740344 1455000 ? D Aug05 121:52 nginx: worker process
nobody 15279 1.3 2.2 9740736 1455484 ? D Aug05 123:00 nginx: worker process
nobody 15281 1.3 2.2 9741164 1455864 ? D Aug05 123:54 nginx: worker process
nobody 15282 1.3 2.2 9738420 1453128 ? D Aug05 123:08 nginx: worker process
nobody 15285 1.3 2.2 9739764 1454368 ? D Aug05 123:42 nginx: worker process
nobody 15286 1.3 2.2 9739528 1454308 ? D Aug05 123:13 nginx: worker process
nobody 15288 1.3 2.2 9739628 1454332 ? D Aug05 124:28 nginx: worker process
nobody 15289 1.3 2.2 9741004 1455664 ? D Aug05 123:23 nginx: worker process
nobody 15290 1.3 2.2 9740816 1455540 ? D Aug05 123:55 nginx: worker process
nobody 15291 1.3 2.2 9739840 1454544 ? D Aug05 122:31 nginx: worker process
nobody 15293 1.3 2.2 9740448 1455316 ? D Aug05 123:45 nginx: worker process
nobody 15294 1.3 2.2 9738528 1453240 ? D Aug05 123:50 nginx: worker process
nobody 15295 1.3 2.2 9739184 1453912 ? D Aug05 123:06 nginx: worker process
nobody 15296 1.3 2.2 9739888 1454664 ? D Aug05 122:55 nginx: worker process
nobody 15298 1.3 2.2 9738940 1453556 ? D Aug05 122:40 nginx: worker process
nobody 15300 1.3 2.2 9740828 1455596 ? D Aug05 122:52 nginx: worker process
nobody 15302 1.3 2.2 9739664 1454412 ? D Aug05 124:54 nginx: worker process
nobody 15303 1.3 2.2 9738036 1452784 ? D Aug05 122:56 nginx: worker process
nobody 15304 1.3 2.2 9741260 1455968 ? D Aug05 123:23 nginx: worker process
nobody 15305 1.3 2.2 9740756 1455660 ? D Aug05 124:17 nginx: worker process
nobody 15306 1.3 2.2 9738720 1453452 ? D Aug05 123:35 nginx: worker process
nobody 15307 1.3 2.2 9738908 1453692 ? D Aug05 124:22 nginx: worker process
nobody 15308 0.0 2.0 9669052 1358768 ? D Aug05 6:31 nginx: cache manager process

--------------------------------------------------------------------------------------------------------------------------------------------------------

[root@w-fc12 conf]# ps aux | awk '$NF=="process" {print " "$0 system("cat /proc/"$2"/wchan")}'
xfs_iget nobody 15268 1.3 2.2 9740068 1454828 ? D Aug05 123:55 nginx: worker process 0
do_lookup nobody 15269 1.3 2.2 9736804 1451504 ? D Aug05 123:06 nginx: worker process 0
do_lookup nobody 15270 1.3 2.2 9738568 1453272 ? D Aug05 123:17 nginx: worker process 0
xfs_iget nobody 15271 1.3 2.2 9739520 1454184 ? D Aug05 123:14 nginx: worker process 0
do_lookup nobody 15272 1.3 2.2 9740440 1455216 ? D Aug05 124:39 nginx: worker process 0
do_lookup nobody 15273 1.3 2.2 9742540 1457160 ? D Aug05 123:52 nginx: worker process 0
xfs_buf_iowait nobody 15274 1.3 2.2 9737832 1452544 ? D Aug05 123:43 nginx: worker process 0
do_lookup nobody 15275 1.3 2.2 9741376 1456092 ? D Aug05 122:14 nginx: worker process 0
do_lookup nobody 15276 1.3 2.2 9738660 1453544 ? D Aug05 122:46 nginx: worker process 0
do_lookup nobody 15277 1.3 2.2 9740412 1455124 ? D Aug05 123:37 nginx: worker process 0
do_lookup nobody 15278 1.3 2.2 9740344 1455000 ? D Aug05 121:52 nginx: worker process 0
xfs_iget nobody 15279 1.3 2.2 9740736 1455484 ? D Aug05 123:00 nginx: worker process 0
xfs_iget nobody 15281 1.3 2.2 9741164 1455864 ? D Aug05 123:54 nginx: worker process 0
do_lookup nobody 15282 1.3 2.2 9738420 1453128 ? D Aug05 123:08 nginx: worker process 0
do_lookup nobody 15285 1.3 2.2 9739764 1454368 ? D Aug05 123:42 nginx: worker process 0
do_lookup nobody 15286 1.3 2.2 9739528 1454308 ? D Aug05 123:13 nginx: worker process 0
do_lookup nobody 15288 1.3 2.2 9739628 1454332 ? D Aug05 124:28 nginx: worker process 0
do_lookup nobody 15289 1.3 2.2 9741004 1455664 ? D Aug05 123:23 nginx: worker process 0
do_lookup nobody 15290 1.3 2.2 9740816 1455540 ? D Aug05 123:55 nginx: worker process 0
do_lookup nobody 15291 1.3 2.2 9739840 1454544 ? D Aug05 122:31 nginx: worker process 0
do_lookup nobody 15293 1.3 2.2 9740448 1455316 ? D Aug05 123:45 nginx: worker process 0
do_lookup nobody 15294 1.3 2.2 9738528 1453240 ? D Aug05 123:50 nginx: worker process 0
xfs_iget nobody 15295 1.3 2.2 9739184 1453912 ? D Aug05 123:06 nginx: worker process 0
do_lookup nobody 15296 1.3 2.2 9739888 1454664 ? D Aug05 122:55 nginx: worker process 0
do_lookup nobody 15298 1.3 2.2 9738940 1453556 ? D Aug05 122:40 nginx: worker process 0
xfs_iget nobody 15300 1.3 2.2 9740828 1455596 ? D Aug05 122:52 nginx: worker process 0
do_lookup nobody 15302 1.3 2.2 9739664 1454412 ? D Aug05 124:54 nginx: worker process 0
do_lookup nobody 15303 1.3 2.2 9738036 1452784 ? D Aug05 122:56 nginx: worker process 0
xfs_iget nobody 15304 1.3 2.2 9741260 1455968 ? D Aug05 123:23 nginx: worker process 0
do_lookup nobody 15305 1.3 2.2 9740756 1455660 ? D Aug05 124:17 nginx: worker process 0
do_lookup nobody 15306 1.3 2.2 9738720 1453452 ? D Aug05 123:35 nginx: worker process 0
xfs_iget nobody 15307 1.3 2.2 9738908 1453692 ? D Aug05 124:22 nginx: worker process 0
do_lookup nobody 15308 0.0 2.0 9669052 1358768 ? D Aug05 6:31 nginx: cache manager process0

--------------------------------------------------------------------------------------------------------------------------------------------------------

[root@w-fc12 conf]# ps aux | awk '$NF=="process" {print " "$0 system("cat /proc/"$2"/syscall")}'
2 0x1aa3240 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4f78 0x343580f000
nobody 15268 1.3 2.2 9740068 1454828 ? D Aug05 123:55 nginx: worker process 0
2 0x20770e0 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4dd0 0x343580f000
nobody 15269 1.3 2.2 9736804 1451504 ? D Aug05 123:06 nginx: worker process 0
2 0x1d94928 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f49f0 0x343580f000
nobody 15270 1.3 2.2 9738568 1453272 ? D Aug05 123:17 nginx: worker process 0
2 0x2c4c270 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f5078 0x343580f000
nobody 15271 1.3 2.2 9739520 1454184 ? D Aug05 123:14 nginx: worker process 0
2 0x3a01228 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f5130 0x343580f000
nobody 15272 1.3 2.2 9740440 1455216 ? D Aug05 124:39 nginx: worker process 0
2 0x3835a00 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4f08 0x343580f000
nobody 15273 1.3 2.2 9742540 1457160 ? D Aug05 123:52 nginx: worker process 0
17 0x298 0x1fac850 0x1000 0x0 0x343578fee8 0x1 0x7fffbb8f4d60 0x343580f0c3
nobody 15274 1.3 2.2 9737832 1452544 ? D Aug05 123:43 nginx: worker process 0
2 0x2d2dc08 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4b10 0x343580f000
nobody 15275 1.3 2.2 9741376 1456092 ? D Aug05 122:14 nginx: worker process 0
2 0x1d99bc0 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4e88 0x343580f000
nobody 15276 1.3 2.2 9738660 1453544 ? D Aug05 122:46 nginx: worker process 0
2 0x12a0430 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4dd0 0x343580f000
nobody 15277 1.3 2.2 9740412 1455124 ? D Aug05 123:37 nginx: worker process 0
2 0x364ed98 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4cb0 0x343580f000
nobody 15278 1.3 2.2 9740344 1455000 ? D Aug05 121:52 nginx: worker process 0
2 0x3a1cea8 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4b20 0x343580f000
nobody 15279 1.3 2.2 9740736 1455484 ? D Aug05 123:00 nginx: worker process 0
2 0x3454218 0x800 0x0 0x7fffffffffffffff 0x0 0x27d3568 0x7fffbb8f4cb0 0x343580f000
nobody 15281 1.3 2.2 9741164 1455864 ? D Aug05 123:54 nginx: worker process 0
2 0x35bc1f8 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4e98 0x343580f000
nobody 15282 1.3 2.2 9738420 1453128 ? D Aug05 123:08 nginx: worker process 0
2 0x2cb4118 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4ee0 0x343580f000
nobody 15285 1.3 2.2 9739764 1454368 ? D Aug05 123:42 nginx: worker process 0
2 0x31b7c20 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4d38 0x343580f000
nobody 15286 1.3 2.2 9739528 1454308 ? D Aug05 123:13 nginx: worker process 0
2 0x29ad468 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4e98 0x343580f000
nobody 15288 1.3 2.2 9739628 1454332 ? D Aug05 124:28 nginx: worker process 0
2 0x14cf388 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4ee0 0x343580f000
nobody 15289 1.3 2.2 9741004 1455664 ? D Aug05 123:23 nginx: worker process 0
2 0x36ef640 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4b38 0x343580f000
nobody 15290 1.3 2.2 9740816 1455540 ? D Aug05 123:55 nginx: worker process 0
2 0x18980d0 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4b90 0x343580f000
nobody 15291 1.3 2.2 9739840 1454544 ? D Aug05 122:31 nginx: worker process 0
2 0x35bf950 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4c90 0x343580f000
nobody 15293 1.3 2.2 9740448 1455316 ? D Aug05 123:45 nginx: worker process 0
2 0x333f660 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f46b8 0x343580f000
nobody 15294 1.3 2.2 9738528 1453240 ? D Aug05 123:50 nginx: worker process 0
2 0x335b0a8 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4e98 0x343580f000
nobody 15295 1.3 2.2 9739184 1453912 ? D Aug05 123:06 nginx: worker process 0
2 0x2814978 0x800 0x0 0x7fffffffffffffff 0x0 0x3f832d0 0x7fffbb8f4d10 0x343580f000
nobody 15296 1.3 2.2 9739888 1454664 ? D Aug05 122:55 nginx: worker process 0
2 0x2049af8 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4ac0 0x343580f000
nobody 15298 1.3 2.2 9738940 1453556 ? D Aug05 122:40 nginx: worker process 0
2 0x1806d60 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f5078 0x343580f000
nobody 15300 1.3 2.2 9740828 1455596 ? D Aug05 122:52 nginx: worker process 0
2 0x2767718 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f48d0 0x343580f000
nobody 15302 1.3 2.2 9739664 1454412 ? D Aug05 124:54 nginx: worker process 0
2 0x2f80b88 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4d40 0x343580f000
nobody 15303 1.3 2.2 9738036 1452784 ? D Aug05 122:56 nginx: worker process 0
2 0x2b13ca8 0x800 0x0 0x7fffffffffffffff 0x0 0x1895888 0x7fffbb8f4920 0x343580f000
nobody 15304 1.3 2.2 9741260 1455968 ? D Aug05 123:23 nginx: worker process 0
2 0x24bd310 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4e30 0x343580f000
nobody 15305 1.3 2.2 9740756 1455660 ? D Aug05 124:17 nginx: worker process 0
2 0x21192b0 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4af0 0x343580f000
nobody 15306 1.3 2.2 9738720 1453452 ? D Aug05 123:35 nginx: worker process 0
2 0x231cbf0 0xc2 0x180 0x0 0x0 0x0 0x7fffbb8f4e88 0x343580f000
nobody 15307 1.3 2.2 9738908 1453692 ? D Aug05 124:22 nginx: worker process 0
87 0xff57a0 0xff57e0 0x0 0x43 0x0 0xfff 0x7fffbb8f4dc8 0x34354dc697
nobody 15308 0.0 2.0 9669052 1358768 ? D Aug05 6:31 nginx: cache manager process0
------------------------------------------------------------------------------------------------------------------------------------------------