Трафик на интерфейсах - не то чтобы совсем выдающийся. Гигабит, полюбому, забить не могли никак. Все netstat-ы, vmstat-ы и прочее - уже все облазил сам. Вот как-то так это выглядит:

Нормальная работа, около 6 тыс. rps, сторона "клиентов":

[root@pf186r /begun/nginx/logs]# netstat -w 1 -I net0 -h
input (net0) output
packets errs idrops bytes packets errs bytes colls
32K 0 0 7.8M 29K 0 13M 0
31K 0 0 7.3M 27K 0 12M 0
31K 0 0 7.4M 27K 0 12M 0

Нормальная работа, сторона "бекендов":

[root@pf186r /begun/nginx/logs]# netstat -w 1 -I net1 -h
input (net1) output
packets errs idrops bytes packets errs bytes colls
25K 0 0 6.4M 24K 0 2.5M 0
24K 0 0 6.3M 23K 0 2.4M 0
25K 0 0 6.5M 24K 0 2.5M 0

А вот, когда включаю нагрузку на одно крыло (около 12 тыс. rps):

Сторона "клиентов:

netstat -w 1 -I net0 -h
input (net0) output
packets errs idrops bytes packets errs bytes colls
56K 0 0 13M 51K 0 23M 0
55K 0 0 13M 51K 0 22M 0
55K 0 0 13M 51K 0 23M 0

Сторона "бекендов":

netstat -w 1 -I net1 -h
input (net1) output
packets errs idrops bytes packets errs bytes colls
33K 0 0 10M 31K 0 3.4M 0
32K 0 0 10M 30K 0 3.2M 0
35K 0 0 11M 32K 0 3.4M 0

Top -S (у меня 4 потока net.isr настроено):

last pid: 5027; load averages: 1.99, 1.99, 1.96 up 0+03:06:47 20:10:20
315 processes: 41 running, 163 sleeping, 111 waiting
CPU: 1.6% user, 0.0% nice, 4.3% system, 5.7% interrupt, 88.4% idle
Mem: 129M Active, 68M Inact, 943M Wired, 132K Cache, 633M Buf, 14G Free
Swap: 8192M Total, 8192M Free

PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU COMMAND
11 root 32 171 ki31 0K 512K CPU0 0 89.5H 2938.57% idle
12 root 114 -44 - 0K 1824K WAIT 8 257:01 173.54% intr
4822 nobody 1 54 0 19736K 16700K kqread 18 2:15 15.58% nginx-0.8.50
4820 nobody 1 53 0 15640K 12088K CPU10 10 1:16 15.28% nginx-0.8.50
4823 nobody 1 50 0 13592K 11308K kqread 23 1:23 12.50% nginx-0.8.50
4826 nobody 1 50 0 13592K 11188K kqread 18 1:23 12.35% nginx-0.8.50
4817 nobody 1 51 0 15640K 12476K CPU27 27 1:47 12.16% nginx-0.8.50
4815 nobody 1 51 0 19736K 15988K CPU3 3 1:55 11.57% nginx-0.8.50
4814 nobody 1 50 0 15640K 12644K kqread 17 2:09 11.47% nginx-0.8.50
4816 nobody 1 50 0 15640K 14628K kqread 12 1:59 11.28% nginx-0.8.50
4825 nobody 1 50 0 13592K 11188K kqread 24 1:30 11.18% nginx-0.8.50
4827 nobody 1 51 0 13592K 10956K kqread 9 1:10 10.89% nginx-0.8.50
4829 nobody 1 49 0 19736K 17444K CPU26 26 3:00 10.16% nginx-0.8.50
4821 nobody 1 49 0 13592K 10976K kqread 18 1:18 9.38% nginx-0.8.50
4824 nobody 1 50 0 25880K 19672K kqread 20 2:21 9.08% nginx-0.8.50
4819 nobody 1 49 0 13592K 10956K kqread 15 1:16 8.79% nginx-0.8.50
4828 nobody 1 48 0 15640K 13760K kqread 19 1:31 8.50% nginx-0.8.50
4818 nobody 1 48 0 13592K 10960K CPU28 28 1:34 8.40% nginx-0.8.50

vmstat -z:

ITEM SIZE LIMIT USED FREE REQUESTS FAILURES

UMA Kegs: 208, 0, 103, 16, 103, 0
UMA Zones: 1216, 0, 103, 2, 103, 0
UMA Slabs: 568, 0, 4854, 11, 8709, 0
UMA RCntSlabs: 568, 0, 25669, 0, 25669, 0
UMA Hash: 256, 0, 1, 14, 3, 0
16 Bucket: 152, 0, 167, 8, 167, 0
32 Bucket: 280, 0, 409, 11, 409, 0
64 Bucket: 536, 0, 998, 3, 998, 55
128 Bucket: 1048, 0, 1915, 2, 1915, 435
VM OBJECT: 216, 0, 2624, 4396, 136752, 0
MAP: 232, 0, 7, 25, 7, 0
KMAP ENTRY: 120, 530937, 52, 3048, 12516, 0
MAP ENTRY: 120, 0, 870, 4152, 322953, 0
DP fakepg: 120, 0, 0, 0, 0, 0
SG fakepg: 120, 0, 0, 0, 0, 0
mt_zone: 2056, 0, 239, 8, 239, 0
16: 16, 0, 3890, 5518, 152996274, 0
32: 32, 0, 5883, 4015, 76303956, 0
64: 64, 0, 4205, 4307, 126508693, 0
128: 128, 0, 12381, 7948, 77255019, 0
256: 256, 0, 1156, 3074, 136053, 0
512: 512, 0, 1385, 1240, 37629, 0
1024: 1024, 0, 126, 1214, 26224, 0
2048: 2048, 0, 138, 972, 3779, 0
4096: 4096, 0, 474, 1598, 18134, 0
Files: 80, 0, 3957, 4098, 76547190, 0
TURNSTILE: 136, 0, 1665, 255, 1665, 0
umtx pi: 96, 0, 0, 0, 0, 0
MAC labels: 40, 0, 0, 0, 0, 0
PROC: 1120, 0, 65, 919, 5028, 0
THREAD: 984, 0, 1038, 626, 1038, 0
SLEEPQUEUE: 80, 0, 1665, 452, 1665, 0
VMSPACE: 392, 0, 45, 1195, 5007, 0
cpuset: 72, 0, 56, 94, 56, 0
audit_record: 952, 0, 0, 0, 0, 0
mbuf_packet: 256, 0, 21378, 23678, 718672899, 0
mbuf: 256, 0, 18868, 7506, 836795541, 0
mbuf_cluster: 2048, 262144, 45056, 6, 45056, 0
mbuf_jumbo_page: 4096, 204800, 383, 2755, 25958114, 0
mbuf_jumbo_9k: 9216, 25600, 0, 0, 0, 0
mbuf_jumbo_16k: 16384, 12800, 0, 0, 0, 0
mbuf_ext_refcnt: 4, 0, 0, 4032, 527, 0
g_bio: 232, 0, 0, 2160, 501394, 0
ttyinq: 160, 0, 135, 345, 315, 0
ttyoutq: 256, 0, 72, 243, 168, 0
ata_request: 320, 0, 0, 0, 0, 0
ata_composite: 336, 0, 0, 0, 0, 0
VNODE: 472, 0, 1984, 1720, 3415, 0
VNODEPOLL: 112, 0, 0, 396, 16, 0
NAMEI: 1024, 0, 0, 1040, 2616775, 0
S VFS Cache: 108, 0, 2181, 1944, 412977, 0
L VFS Cache: 328, 0, 3, 633, 233, 0
DIRHASH: 1024, 0, 54, 42, 54, 0
pipe: 728, 0, 2, 848, 3934, 0
ksiginfo: 112, 0, 788, 2446, 10082, 0
itimer: 344, 0, 1, 21, 1, 0
KNOTE: 128, 0, 6622, 5819, 904967024, 0
socket: 680, 204804, 4416, 3330, 76392941, 0
ipq: 56, 8253, 0, 0, 0, 0
udp_inpcb: 336, 204809, 25, 910, 2643, 0
udpcb: 16, 204960, 25, 4007, 2643, 0
tcp_inpcb: 336, 204809, 4368, 3497, 76389212, 0
tcpcb: 880, 204800, 4342, 3342, 76389212, 0
tcptw: 72, 81950, 26, 4974, 14429, 0
syncache: 144, 491530, 1, 2573, 340071, 0
hostcache: 136, 15372, 63, 581, 99, 0
tcpreass: 40, 16464, 0, 2268, 16823, 0
sackhole: 32, 0, 0, 3131, 2270, 0
sctp_ep: 1272, 25602, 0, 0, 0, 0
sctp_asoc: 2232, 40000, 0, 0, 0, 0
sctp_laddr: 48, 80064, 0, 216, 8, 0
sctp_raddr: 600, 80004, 0, 0, 0, 0
sctp_chunk: 136, 400008, 0, 0, 0, 0
sctp_readq: 104, 400032, 0, 0, 0, 0
sctp_stream_msg_out: 96, 400026, 0, 0, 0, 0
sctp_asconf: 40, 400008, 0, 0, 0, 0
sctp_asconf_ack: 48, 400032, 0, 0, 0, 0
ripcb: 336, 204809, 0, 0, 0, 0
unpcb: 240, 204800, 44, 1012, 1080, 0
rtentry: 200, 0, 25, 431, 116, 0
pfsrctrpl: 152, 0, 0, 0, 0, 0
pfrulepl: 912, 0, 0, 0, 0, 0
pfstatepl: 392, 10000, 0, 0, 0, 0
pfaltqpl: 240, 0, 0, 0, 0, 0
pfpooladdrpl: 88, 0, 0, 0, 0, 0
pfrktable: 1296, 0, 0, 0, 0, 0
pfrkentry: 216, 0, 0, 0, 0, 0
pfrkentry2: 216, 0, 0, 0, 0, 0
pffrent: 32, 5050, 0, 0, 0, 0
pffrag: 80, 0, 0, 0, 0, 0
pffrcache: 80, 10035, 0, 0, 0, 0
pffrcent: 24, 50022, 0, 0, 0, 0
pfstatescrub: 40, 0, 0, 0, 0, 0
pfiaddrpl: 120, 0, 0, 0, 0, 0
pfospfen: 112, 0, 0, 0, 0, 0
pfosfp: 40, 0, 0, 0, 0, 0
selfd: 56, 0, 394, 2441, 683658, 0
SWAPMETA: 288, 116519, 0, 0, 0, 0
ip4flow: 56, 787500, 0, 0, 0, 0
ip6flow: 80, 787500, 0, 0, 0, 0
Mountpoints: 752, 0, 10, 15, 10, 0
FFS inode: 168, 0, 1938, 1670, 3365, 0
FFS1 dinode: 128, 0, 0, 0, 0, 0
FFS2 dinode: 256, 0, 1938, 1707, 3365, 0
md0: 512, 0, 301, 28, 301, 0

netstat -m:

37596/33834/71430 mbufs in use (current/cache/total)
18851/26211/45062/262144 mbuf clusters in use (current/cache/total/max)
18851/26205 mbuf+clusters out of packet secondary zone in use (current/cache)
305/2833/3138/204800 4k (page size) jumbo clusters in use (current/cache/total/max)
0/0/0/25600 9k jumbo clusters in use (current/cache/total/max)
0/0/0/12800 16k jumbo clusters in use (current/cache/total/max)
48321K/72212K/120533K bytes allocated to network (current/cache/total)
0/0/0 requests for mbufs denied (mbufs/clusters/mbuf+clusters)
0/0/0 requests for jumbo clusters denied (4k/9k/16k)
0/0/0 sfbufs in use (current/peak/max)
0 requests for sfbufs denied
0 requests for sfbufs delayed
1 requests for I/O initiated by sendfile
0 calls to protocol drain routines

netstat -s -p tcp

tcp:
575984102 packets sent
199793966 data packets (128240461977 bytes)
120548 data packets (109908884 bytes) retransmitted
6298 data packets unnecessarily retransmitted
0 resends initiated by MTU discovery
211031186 ack-only packets (699696 delayed)
0 URG only packets
0 window probe packets
12233847 window update packets
153290173 control packets
636296299 packets received
328855333 acks (for 128180931469 bytes)
50188015 duplicate acks
0 acks for unsent data
319637824 packets (118077153792 bytes) received in-sequence
91965 completely duplicate packets (4352651 bytes)
0 old duplicate packets
0 packets with some dup. data (0 bytes duped)
16973 out-of-order packets (9523463 bytes)
0 packets (0 bytes) of data after window
0 window probes
323 window update packets
9450 packets received after close
0 discarded for bad checksums
0 discarded for bad header offset fields
0 discarded because packet too short
0 discarded due to memory problems
76678121 connection requests
336348 connection accepts
0 bad connection attempts
0 listen queue overflows
67129 ignored RSTs in the windows
76615375 connections established (including accepts)
77012185 connections closed (including 15151 drops)
305966 connections updated cached RTT on close
314983 connections updated cached RTT variance on close
242827 connections updated cached ssthresh on close
93 embryonic connections dropped
328675930 segments updated rtt (of 305869103 attempts)
160120 retransmit timeouts
9805 connections dropped by rexmit timeout
0 persist timeouts
0 connections dropped by persist timeout
2 Connections (fin_wait_2) dropped because of timeout
93 keepalive timeouts
0 keepalive probes sent
93 connections dropped by keepalive
5610453 correct ACK header predictions
225822463 correct data packet header predictions
341939 syncache entries added
997 retransmitted
0 dupsyn
0 dropped
336348 completed
0 bucket overflow
0 cache overflow
3339 reset
319 stale
0 aborted
0 badack
0 unreach
0 zone failures
341939 cookies sent
0 cookies received
123 SACK recovery episodes
5 segment rexmits in SACK recovery episodes
6128 byte rexmits in SACK recovery episodes
13642 SACK options (SACK blocks) received
7557 SACK options (SACK blocks) sent
0 SACK scoreboard overflow
0 packets with ECN CE bit set
0 packets with ECN ECT(0) bit set
0 packets with ECN ECT(1) bit set
0 successful ECN handshakes
0 times ECN reduced the congestion window