aio sendfile в nginx'е использует флаг SF_NODISKIO и sendfile() не
блокируется на диске, а сообщает об отсутствии данных в памяти,
после чего nginx инициирует асинхронную подгрузку данных, читая
только один байт. При этом ядро FreeBSD подгружает в память первые
128K файла, однако при последующих чтениях файл подгружается частями
только по 16K.

К письму прилагается два патча

1) для FreeBSD 7, реализующий fcntl(F_RDAHEAD, размер):

patch -d /usr/src < patch.rdahead
после чего пересобираем и устанавливаем ядро и перегружаемся.

После загрузки обновляем fcntl.h:

cd /usr/src/include/
make obj all install

2) для nginx-0.8.15 - patch.read_ahead

Настройки:

sendfile on;
aio sendfile;
read_ahead 256k;
tcp_nopush on;

sysctl vfs.read_max=16 # 256K

Для Линукса read_ahead с любым размером >0 вызывает просто

posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)


--
Игорь Сысоев
http://sysoev.ru
--- sys/sys/fcntl.h 2009-06-02 19:05:17.000000000 +0400
+++ sys/sys/fcntl.h 2009-09-12 20:29:34.000000000 +0400
@@ -118,6 +118,10 @@
#if __BSD_VISIBLE
/* Attempt to bypass buffer cache */
#define O_DIRECT 0x00010000
+#ifdef _KERNEL
+/* Read ahead */
+#define O_RDAHEAD 0x00020000
+#endif
#endif

/*
@@ -187,6 +191,7 @@
#define F_SETLK 12 /* set record locking information */
#define F_SETLKW 13 /* F_SETLK; wait if blocked */
#define F_SETLK_REMOTE 14 /* debugging support for remote locks */
+#define F_RDAHEAD 15 /* read ahead */

/* file descriptor flags (F_GETFD, F_SETFD) */
#define FD_CLOEXEC 1 /* close-on-exec flag */
--- sys/kern/vfs_vnops.c 2009-06-02 19:05:00.000000000 +0400
+++ sys/kern/vfs_vnops.c 2009-09-12 20:24:00.000000000 +0400
@@ -305,6 +305,9 @@
sequential_heuristic(struct uio *uio, struct file *fp)
{

+ if (fp->f_flag & O_RDAHEAD)
+ return(fp->f_seqcount << IO_SEQSHIFT);
+
if ((uio->uio_offset == 0 && fp->f_seqcount > 0) ||
uio->uio_offset == fp->f_nextoff) {
/*
--- sys/kern/kern_descrip.c 2009-08-28 18:50:11.000000000 +0400
+++ sys/kern/kern_descrip.c 2009-09-12 20:23:36.000000000 +0400
@@ -411,6 +411,7 @@
u_int newmin;
int error, flg, tmp;
int vfslocked;
+ uint64_t bsize;

vfslocked = 0;
error = 0;
@@ -694,6 +695,31 @@
vfslocked = 0;
fdrop(fp, td);
break;
+
+ case F_RDAHEAD:
+ FILEDESC_SLOCK(fdp);
+ if ((fp = fdtofp(fd, fdp)) == NULL) {
+ FILEDESC_SUNLOCK(fdp);
+ error = EBADF;
+ break;
+ }
+ if (fp->f_type != DTYPE_VNODE) {
+ FILEDESC_SUNLOCK(fdp);
+ error = EBADF;
+ break;
+ }
+ FILE_LOCK(fp);
+ if (arg) {
+ bsize = fp->f_vnode->v_mount->mnt_stat.f_iosize;
+ fp->f_seqcount = (arg + bsize - 1) / bsize;
+ fp->f_flag |= O_RDAHEAD;
+ } else {
+ fp->f_flag &= ~O_RDAHEAD;
+ }
+ FILE_UNLOCK(fp);
+ FILEDESC_SUNLOCK(fdp);
+ break;
+
default:
error = EINVAL;
break;
Index: src/http/ngx_http_core_module.c
===================================================================
--- src/http/ngx_http_core_module.c (revision 2440)
+++ src/http/ngx_http_core_module.c (working copy)
@@ -408,6 +408,13 @@

#endif

+ { ngx_string("read_ahead"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_size_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_core_loc_conf_t, read_ahead),
+ NULL },
+
{ ngx_string("directio"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
ngx_http_core_directio,
@@ -2955,6 +2962,7 @@
#if (NGX_HAVE_FILE_AIO)
lcf->aio = NGX_CONF_UNSET;
#endif
+ lcf->read_ahead = NGX_CONF_UNSET_SIZE;
lcf->directio = NGX_CONF_UNSET;
lcf->directio_alignment = NGX_CONF_UNSET;
lcf->tcp_nopush = NGX_CONF_UNSET;
@@ -3156,6 +3164,7 @@
#if (NGX_HAVE_FILE_AIO)
ngx_conf_merge_value(conf->aio, prev->aio, 0);
#endif
+ ngx_conf_merge_size_value(conf->read_ahead, prev->read_ahead, 0);
ngx_conf_merge_off_value(conf->directio, prev->directio,
NGX_MAX_OFF_T_VALUE);
ngx_conf_merge_off_value(conf->directio_alignment, prev->directio_alignment,
Index: src/http/ngx_http_file_cache.c
===================================================================
--- src/http/ngx_http_file_cache.c (revision 2440)
+++ src/http/ngx_http_file_cache.c (working copy)
@@ -281,6 +281,7 @@
of.min_uses = clcf->open_file_cache_min_uses;
of.events = clcf->open_file_cache_events;
of.directio = NGX_OPEN_FILE_DIRECTIO_OFF;
+ of.read_ahead = clcf->read_ahead;

if (ngx_open_cached_file(clcf->open_file_cache, &c->file.name, &of, r->pool)
!= NGX_OK)
Index: src/http/ngx_http_core_module.h
===================================================================
--- src/http/ngx_http_core_module.h (revision 2440)
+++ src/http/ngx_http_core_module.h (working copy)
@@ -332,6 +332,7 @@
size_t limit_rate; /* limit_rate */
size_t limit_rate_after; /* limit_rate_after */
size_t sendfile_max_chunk; /* sendfile_max_chunk */
+ size_t read_ahead; /* read_ahead */

ngx_msec_t client_body_timeout; /* client_body_timeout */
ngx_msec_t send_timeout; /* send_timeout */
Index: src/http/ngx_http_script.c
===================================================================
--- src/http/ngx_http_script.c (revision 2440)
+++ src/http/ngx_http_script.c (working copy)
@@ -1407,6 +1407,7 @@

ngx_memzero(&of, sizeof(ngx_open_file_info_t));

+ of.read_ahead = clcf->read_ahead;
of.directio = clcf->directio;
of.valid = clcf->open_file_cache_valid;
of.min_uses = clcf->open_file_cache_min_uses;
Index: src/http/modules/ngx_http_gzip_static_module.c
===================================================================
--- src/http/modules/ngx_http_gzip_static_module.c (revision 2440)
+++ src/http/modules/ngx_http_gzip_static_module.c (working copy)
@@ -120,6 +120,7 @@

ngx_memzero(&of, sizeof(ngx_open_file_info_t));

+ of.read_ahead = clcf->read_ahead;
of.directio = clcf->directio;
of.valid = clcf->open_file_cache_valid;
of.min_uses = clcf->open_file_cache_min_uses;
Index: src/http/modules/ngx_http_index_module.c
===================================================================
--- src/http/modules/ngx_http_index_module.c (revision 2440)
+++ src/http/modules/ngx_http_index_module.c (working copy)
@@ -205,6 +205,7 @@

ngx_memzero(&of, sizeof(ngx_open_file_info_t));

+ of.read_ahead = clcf->read_ahead;
of.directio = clcf->directio;
of.valid = clcf->open_file_cache_valid;
of.min_uses = clcf->open_file_cache_min_uses;
Index: src/http/modules/ngx_http_static_module.c
===================================================================
--- src/http/modules/ngx_http_static_module.c (revision 2440)
+++ src/http/modules/ngx_http_static_module.c (working copy)
@@ -91,6 +91,7 @@

ngx_memzero(&of, sizeof(ngx_open_file_info_t));

+ of.read_ahead = clcf->read_ahead;
of.directio = clcf->directio;
of.valid = clcf->open_file_cache_valid;
of.min_uses = clcf->open_file_cache_min_uses;
Index: src/http/modules/ngx_http_flv_module.c
===================================================================
--- src/http/modules/ngx_http_flv_module.c (revision 2440)
+++ src/http/modules/ngx_http_flv_module.c (working copy)
@@ -106,6 +106,7 @@

ngx_memzero(&of, sizeof(ngx_open_file_info_t));

+ of.read_ahead = clcf->read_ahead;
of.directio = clcf->directio;
of.valid = clcf->open_file_cache_valid;
of.min_uses = clcf->open_file_cache_min_uses;
Index: src/http/modules/perl/nginx.xs
===================================================================
--- src/http/modules/perl/nginx.xs (revision 2440)
+++ src/http/modules/perl/nginx.xs (working copy)
@@ -648,6 +648,7 @@

ngx_memzero(&of, sizeof(ngx_open_file_info_t));

+ of.read_ahead = clcf->read_ahead;
of.directio = clcf->directio;
of.valid = clcf->open_file_cache_valid;
of.min_uses = clcf->open_file_cache_min_uses;
Index: src/os/win32/ngx_files.c
===================================================================
--- src/os/win32/ngx_files.c (revision 2440)
+++ src/os/win32/ngx_files.c (working copy)
@@ -531,6 +531,13 @@


ngx_int_t
+ngx_read_ahead(ngx_fd_t fd, size_t n)
+{
+ return ~NGX_FILE_ERROR;
+}
+
+
+ngx_int_t
ngx_directio_on(ngx_fd_t fd)
{
return 0;
Index: src/os/win32/ngx_files.h
===================================================================
--- src/os/win32/ngx_files.h (revision 2440)
+++ src/os/win32/ngx_files.h (working copy)
@@ -232,6 +232,8 @@
ssize_t ngx_write_chain_to_file(ngx_file_t *file, ngx_chain_t *ce,
off_t offset, ngx_pool_t *pool);

+ngx_int_t ngx_read_ahead(ngx_fd_t fd, size_t n);
+#define ngx_read_ahead_n "ngx_read_ahead_n"

ngx_int_t ngx_directio_on(ngx_fd_t fd);
#define ngx_directio_on_n "ngx_directio_on_n"
Index: src/os/unix/ngx_files.h
===================================================================
--- src/os/unix/ngx_files.h (revision 2440)
+++ src/os/unix/ngx_files.h (working copy)
@@ -254,6 +254,28 @@
#define ngx_unlock_fd_n "fcntl(F_SETLK, F_UNLCK)"


+#if (NGX_HAVE_F_RDAHEAD)
+
+#define NGX_HAVE_READ_AHEAD 1
+
+#define ngx_read_ahead(fd, n) fcntl(fd, F_RDAHEAD, (int) n)
+#define ngx_read_ahead_n "fcntl(fd, F_RDAHEAD)"
+
+#elif (NGX_HAVE_POSIX_FADVISE)
+
+#define NGX_HAVE_READ_AHEAD 1
+
+#define ngx_read_ahead(fd, n) posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)
+#define ngx_read_ahead_n "posix_fadvise(POSIX_FADV_SEQUENTIAL)"
+
+#else
+
+#define ngx_read_ahead(fd, n) 0
+#define ngx_read_ahead_n "ngx_read_ahead_n"
+
+#endif
+
+
#if (NGX_HAVE_O_DIRECT)

ngx_int_t ngx_directio_on(ngx_fd_t fd);
Index: src/core/ngx_open_file_cache.c
===================================================================
--- src/core/ngx_open_file_cache.c (revision 2440)
+++ src/core/ngx_open_file_cache.c (working copy)
@@ -17,6 +17,9 @@
*/


+#define NGX_MIN_READ_AHEAD (128 * 1024)
+
+
static void ngx_open_file_cache_cleanup(void *data);
static ngx_int_t ngx_open_and_stat_file(u_char *name, ngx_open_file_info_t *of,
ngx_log_t *log);
@@ -524,6 +527,13 @@
} else {
of->fd = fd;

+ if (of->read_ahead && ngx_file_size(&fi) > NGX_MIN_READ_AHEAD) {
+ if (ngx_read_ahead(fd, of->read_ahead) == NGX_ERROR) {
+ ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
+ ngx_read_ahead_n " \"%s\" failed", name);
+ }
+ }
+
if (of->directio <= ngx_file_size(&fi)) {
if (ngx_directio_on(fd) == -1) {
ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
Index: src/core/ngx_open_file_cache.h
===================================================================
--- src/core/ngx_open_file_cache.h (revision 2440)
+++ src/core/ngx_open_file_cache.h (working copy)
@@ -21,6 +21,7 @@
time_t mtime;
off_t size;
off_t directio;
+ size_t read_ahead;

ngx_err_t err;
char *failed;
Index: auto/os/features
===================================================================
--- auto/os/features (revision 2440)
+++ auto/os/features (working copy)
@@ -172,6 +172,26 @@
fi


+ngx_feature="F_RDAHEAD"
+ngx_feature_name="NGX_HAVE_F_RDAHEAD"
+ngx_feature_run=no
+ngx_feature_incs="#include <fcntl.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="fcntl(0, F_RDAHEAD, 1);"
+. auto/feature
+
+
+ngx_feature="posix_fadvise()"
+ngx_feature_name="NGX_HAVE_POSIX_FADVISE"
+ngx_feature_run=no
+ngx_feature_incs="#include <fcntl.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="posix_fadvise(0, 0, 0, POSIX_FADV_SEQUENTIAL);"
+. auto/feature
+
+
ngx_feature="O_DIRECT"
ngx_feature_name="NGX_HAVE_O_DIRECT"
ngx_feature_run=no