# HG changeset patch
# User Agile6v <agile6v@agile6v.com>
# Date 1524585905 -28800
# Wed Apr 25 00:05:05 2018 +0800
# Node ID 89793df28d1bcf2baf00e389e6806d32d7435886
# Parent 7c614ef3c6ea330c62630d5065f961a27d0f82cd
The auto parameter of the worker_processes supports to detect the container environment.


Docker mounts cgroup information into container starting with version 1.8,
so it is possible to determine the appropriate number of CPUs based on the
cgroup information in the container. Refer to JDK related implementation:
https://bugs.openjdk.java.net/browse/JDK-8146115


diff -r 7c614ef3c6ea -r 89793df28d1b auto/sources
--- a/auto/sources Wed Apr 18 16:11:41 2018 +0300
+++ b/auto/sources Wed Apr 25 00:05:05 2018 +0800
@@ -192,8 +192,8 @@
FREEBSD_SRCS=src/os/unix/ngx_freebsd_init.c
FREEBSD_SENDFILE_SRCS=src/os/unix/ngx_freebsd_sendfile_chain.c

-LINUX_DEPS="src/os/unix/ngx_linux_config.h src/os/unix/ngx_linux.h"
-LINUX_SRCS=src/os/unix/ngx_linux_init.c
+LINUX_DEPS="src/os/unix/ngx_linux_config.h src/os/unix/ngx_linux.h src/os/unix/ngx_container.h"
+LINUX_SRCS="src/os/unix/ngx_linux_init.c src/os/unix/ngx_container.c"
LINUX_SENDFILE_SRCS=src/os/unix/ngx_linux_sendfile_chain.c


diff -r 7c614ef3c6ea -r 89793df28d1b src/os/unix/ngx_container.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/os/unix/ngx_container.c Wed Apr 25 00:05:05 2018 +0800
@@ -0,0 +1,418 @@
+
+/*
+ * Copyright (C) agile6v
+ * Copyright (C) Xiaomi, Inc.
+ */
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+
+#define NGX_BUFFER_SIZE 8192
+#define PER_CPU_SHARES 1024
+
+typedef struct {
+ ngx_str_t root;
+ ngx_str_t path;
+ ngx_str_t mount_point;
+} cgroup_subsystem_info;
+
+static cgroup_subsystem_info cpu_subsystem;
+
+static ngx_str_t proc_cgroup_file = ngx_string("/proc/self/cgroup");
+static ngx_str_t proc_mount_file = ngx_string("/proc/self/mountinfo");
+static ngx_str_t cpu_cfs_period = ngx_string("/cpu.cfs_period_us");
+static ngx_str_t cpu_cfs_quota = ngx_string("/cpu.cfs_quota_us");
+static ngx_str_t cpu_cfs_shares = ngx_string("/cpu.shares");
+
+static float ngx_ceilf(float x)
+{
+ long r = x;
+
+ if (r < 0) {
+ return r;
+ } else {
+ return (r + ((r < x) ? 1 : 0));
+ }
+}
+
+
+static ngx_int_t
+ngx_set_subsystem_path(cgroup_subsystem_info *subsystem_info,
+ ngx_str_t *cgroup_path, ngx_pool_t *pool)
+{
+ u_char *p;
+ ngx_uint_t len;
+
+ if (subsystem_info->root.len != 0 && cgroup_path->data != NULL) {
+ if (ngx_strcmp(subsystem_info->root.data, "/") == 0) {
+ len = subsystem_info->mount_point.len;
+ if (ngx_strcmp(cgroup_path->data, "/") != 0) {
+ len += cgroup_path->len;
+ }
+
+ if (len > NGX_MAX_PATH) {
+ ngx_log_error(NGX_LOG_WARN, pool->log, 0,
+ "the length of the cgroup path exceeds the maximum " \
+ "length of the path (%d) ", NGX_MAX_PATH);
+ return NGX_ERROR;
+ }
+
+ p = ngx_palloc(pool, len + 1);
+ if (p == NULL) {
+ return NGX_ERROR;
+ }
+
+ subsystem_info->path.data = p;
+ subsystem_info->path.len = len;
+
+ if (ngx_strcmp(cgroup_path->data, "/") != 0) {
+ p = ngx_sprintf(p, "%V%V",
+ &subsystem_info->mount_point,
+ cgroup_path);
+ } else {
+ p = ngx_sprintf(p, "%V", &subsystem_info->mount_point);
+ }
+
+ *p = '\0';
+ } else if (ngx_strcmp(subsystem_info->root.data,
+ cgroup_path->data) == 0)
+ {
+ subsystem_info->path = subsystem_info->mount_point;
+ }
+ }
+
+ return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_read_proc_file(ngx_str_t *filename, ngx_str_t *buf, ngx_pool_t *pool)
+{
+ ngx_int_t ret;
+ ngx_uint_t i;
+ size_t size;
+ ssize_t n;
+ ngx_file_t file;
+
+ ngx_memzero(&file, sizeof(ngx_file_t));
+
+ file.name = *filename;
+ file.log = pool->log;
+ file.fd = ngx_open_file(file.name.data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
+ if (file.fd == NGX_INVALID_FILE) {
+ return NGX_ERROR;
+ }
+
+ ret = NGX_OK;
+
+ // Typically this file will not be too big, then try to read all data
+ for (i = 1; i <= 5; i++) {
+ size = NGX_BUFFER_SIZE * i;
+ buf->data = ngx_palloc(pool, size);
+ if (buf->data == NULL) {
+ ret = NGX_ERROR;
+ break;
+ }
+
+ n = ngx_read_file(&file, buf->data, size, 0);
+ if (n == NGX_ERROR) {
+ ret = NGX_ERROR;
+ break;
+ }
+
+ buf->len = n;
+
+ if (buf->len < size) {
+ break;
+ }
+ }
+
+ if (ngx_close_file(file.fd) == NGX_FILE_ERROR) {
+ ngx_log_error(NGX_LOG_ALERT, file.log, ngx_errno,
+ ngx_close_file_n " \"%s\" failed", file.name);
+ return NGX_ERROR;
+ }
+
+ return ret;
+}
+
+
+/*
+ * refers to http://man7.org/linux/man-pages/man7/cgroups.7.html
+ */
+ngx_int_t
+ngx_parse_cgroup_file(ngx_pool_t *pool)
+{
+ ngx_int_t ret;
+ ngx_str_t buf;
+ ngx_str_t fields[3];
+ ngx_uint_t i, index;
+ u_char *start, *end, end_of_line;
+
+ ret = ngx_read_proc_file(&proc_cgroup_file, &buf, pool);
+ if (ret == NGX_ERROR) {
+ return ret;
+ }
+
+ index = 0;
+ start = end = buf.data;
+
+ for (i = 0; i < buf.len; i++) {
+ if (*end == ':' || *end == '\n') {
+ end_of_line = (*end == '\n') ? 1 : 0;
+ *end = '\0';
+
+ fields[index].data = start;
+ fields[index].len = end - start;
+
+ if (end_of_line) {
+ index = 0;
+ if (ngx_strstr(fields[1].data, "cpu,cpuacct") != NULL) {
+ ret = ngx_set_subsystem_path(&cpu_subsystem, &fields[2], pool);
+ if (ret == NGX_ERROR) {
+ return ret;
+ }
+ }
+ } else {
+ index++;
+ }
+
+ start = end + 1;
+ }
+
+ end++;
+ }
+
+ return NGX_OK;
+}
+
+
+/*
+ * refers to http://man7.org/linux/man-pages/man5/proc.5.html
+ */
+ngx_int_t
+ngx_parse_mount_info_file(ngx_pool_t *pool)
+{
+ ngx_str_t buf;
+ ngx_str_t fields[11];
+ ngx_int_t ret;
+ ngx_uint_t i, index, fs_type_index;
+ u_char *start, *end, end_of_line;
+
+ ret = ngx_read_proc_file(&proc_mount_file, &buf, pool);
+ if (ret == NGX_ERROR) {
+ return ret;
+ }
+
+ fs_type_index = index = 0;
+ start = end = buf.data;
+
+ for (i = 0; i < buf.len; i++) {
+ if (*end == ' ' || *end == '\n') {
+ end_of_line = (*end == '\n') ? 1 : 0;
+ *end = '\0';
+
+ fields[index].data = start;
+ fields[index].len = end - start;
+
+ if (fields[index].len == 1 && *fields[index].data == '-') {
+ fs_type_index = index + 1;
+ }
+
+ if (end_of_line) {
+ index = 0;
+ if (fields[fs_type_index].len == 6
+ && ngx_strcmp(fields[fs_type_index].data, "cgroup") == 0
+ && ngx_strstr(fields[4].data, "cpu,cpuacct") != NULL)
+ {
+ cpu_subsystem.root = fields[3];
+ cpu_subsystem.mount_point = fields[4];
+ }
+ } else {
+ index++;
+ }
+
+ start = end + 1;
+ }
+
+ end++;
+ }
+
+ return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_read_subsystem_info(cgroup_subsystem_info *subsystem,
+ ngx_str_t *filename, ngx_log_t *log, ngx_int_t *value)
+{
+ ngx_file_t file;
+ ngx_int_t ret;
+ ngx_str_t full_path;
+ ssize_t n;
+ u_char *p;
+ u_char sign = 0;
+ u_char buf[NGX_MAX_PATH + 1];
+
+ if (subsystem->path.data == NULL || subsystem->path.len == 0) {
+ return NGX_DONE;
+ }
+
+ if ((subsystem->path.len + filename->len) > NGX_MAX_PATH) {
+ return NGX_DONE;
+ }
+
+ full_path.data = buf;
+ full_path.len = NGX_MAX_PATH;
+
+ ngx_memzero(&file, sizeof(ngx_file_t));
+
+ p = ngx_sprintf(full_path.data, "%V%V", &subsystem->path, filename);
+ *p = '\0';
+ full_path.len = p - full_path.data;
+
+ file.name = full_path;
+ file.log = log;
+ file.fd = ngx_open_file(file.name.data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0);
+ if (file.fd == NGX_INVALID_FILE) {
+ return NGX_ERROR;
+ }
+
+ n = ngx_read_file(&file, buf, NGX_BUFFER_SIZE, 0);
+ if (n == NGX_ERROR) {
+ return n;
+ }
+
+ if (ngx_close_file(file.fd) == NGX_FILE_ERROR) {
+ ngx_log_error(NGX_LOG_ALERT, file.log, ngx_errno,
+ ngx_close_file_n " \"%s\" failed", file.name);
+ return NGX_ERROR;
+ }
+
+ n = (n > 0) ? n - 1 : n; // remove linefeed
+ if (n == 0) {
+ return NGX_DONE;
+ }
+
+ p = buf;
+ if (*p == '-') {
+ sign = 1;
+ p++;
+ n--;
+ }
+
+ ret = ngx_atoi(p, n);
+ if (ret == NGX_ERROR) {
+ return NGX_DONE;
+ }
+
+ if (sign) {
+ *value = -ret;
+ } else {
+ *value = ret;
+ }
+
+ return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_get_active_processor_count(ngx_log_t *log) {
+ ngx_int_t quota_count = 0, share_count = 0;
+ ngx_int_t cpu_count, limit_count;
+ ngx_int_t ret, quota, period, shares;
+ ngx_cpuset_t cpu_affinity;
+
+ ret = ngx_getaffinity(&cpu_affinity, log);
+ if (ret == NGX_ERROR) {
+ return ret;
+ }
+
+ cpu_count = limit_count = ret;
+
+ ngx_log_error(NGX_LOG_INFO, log, 0,
+ "active processor count: %d", cpu_count);
+
+ ret = ngx_read_subsystem_info(&cpu_subsystem,
+ &cpu_cfs_shares, log, &shares);
+ if (ret == NGX_DONE || ret == NGX_ERROR) {
+ return ret;
+ }
+
+ ret = ngx_read_subsystem_info(&cpu_subsystem,
+ &cpu_cfs_quota, log, &quota);
+ if (ret == NGX_DONE || ret == NGX_ERROR) {
+ return ret;
+ }
+
+ ret = ngx_read_subsystem_info(&cpu_subsystem,
+ &cpu_cfs_period, log, &period);
+ if (ret == NGX_DONE || ret == NGX_ERROR) {
+ return ret;
+ }
+
+ ngx_log_error(NGX_LOG_INFO, log, 0,
+ "cgroup cpu subsystem: quota=%d, period=%d, shares=%d",
+ quota, period, shares);
+
+ if (shares == PER_CPU_SHARES) {
+ shares = -1;
+ }
+
+ if (quota > -1 && period > 0) {
+ quota_count = ngx_ceilf((float) quota / (float) period);
+ }
+
+ if (shares > -1) {
+ share_count = ngx_ceilf((float) shares / (float) PER_CPU_SHARES);
+ }
+
+ if (quota_count != 0 && share_count != 0) {
+ limit_count = ngx_min(quota_count, share_count);
+ } else if (quota_count != 0) {
+ limit_count = quota_count;
+ } else if (share_count != 0) {
+ limit_count = share_count;
+ }
+
+ return ngx_min(cpu_count, limit_count);
+}
+
+
+ngx_int_t
+ngx_container_init(ngx_log_t *log)
+{
+ ngx_pool_t *pool;
+ ngx_int_t ret;
+
+ pool = ngx_create_pool(NGX_DEFAULT_POOL_SIZE, log);
+ if (pool == NULL) {
+ return NGX_ERROR;
+ }
+
+ pool->log = log;
+
+ do {
+ ret = ngx_parse_mount_info_file(pool);
+ if (ret != NGX_OK) {
+ break;
+ }
+
+ ret = ngx_parse_cgroup_file(pool);
+ if (ret != NGX_OK) {
+ break;
+ }
+
+ if (cpu_subsystem.path.data == NULL || cpu_subsystem.path.len == 0) {
+ ret = NGX_DONE;
+ break;
+ }
+
+ ret = ngx_get_active_processor_count(log);
+
+ } while (0);
+
+ ngx_destroy_pool(pool);
+
+ return ret;
+}
diff -r 7c614ef3c6ea -r 89793df28d1b src/os/unix/ngx_container.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/os/unix/ngx_container.h Wed Apr 25 00:05:05 2018 +0800
@@ -0,0 +1,12 @@
+
+/*
+ * Copyright (C) agile6v
+ * Copyright (C) Xiaomi, Inc.
+ */
+
+#ifndef _NGX_CONTAINER_H_INCLUDED_
+#define _NGX_CONTAINER_H_INCLUDED_
+
+ngx_int_t ngx_container_init(ngx_log_t *log);
+
+#endif /* _NGX_CONTAINER_H_INCLUDED_ */
diff -r 7c614ef3c6ea -r 89793df28d1b src/os/unix/ngx_linux_init.c
--- a/src/os/unix/ngx_linux_init.c Wed Apr 18 16:11:41 2018 +0300
+++ b/src/os/unix/ngx_linux_init.c Wed Apr 25 00:05:05 2018 +0800
@@ -7,12 +7,12 @@

#include <ngx_config.h>
#include <ngx_core.h>
+#include <ngx_container.h>


u_char ngx_linux_kern_ostype[50];
u_char ngx_linux_kern_osrelease[50];

-
static ngx_os_io_t ngx_linux_io = {
ngx_unix_recv,
ngx_readv_chain,
@@ -33,6 +33,7 @@
ngx_int_t
ngx_os_specific_init(ngx_log_t *log)
{
+ ngx_int_t ret;
struct utsname u;

if (uname(&u) == -1) {
@@ -48,6 +49,13 @@

ngx_os_io = ngx_linux_io;

+ ret = ngx_container_init(log);
+ if (ret == NGX_ERROR) {
+ return ret;
+ } else if (ret > NGX_OK) {
+ ngx_ncpu = ret;
+ }
+
return NGX_OK;
}

diff -r 7c614ef3c6ea -r 89793df28d1b src/os/unix/ngx_setaffinity.c
--- a/src/os/unix/ngx_setaffinity.c Wed Apr 18 16:11:41 2018 +0300
+++ b/src/os/unix/ngx_setaffinity.c Wed Apr 25 00:05:05 2018 +0800
@@ -30,6 +30,12 @@
}
}

+ngx_int_t
+ngx_getaffinity(ngx_cpuset_t *cpu_affinity, ngx_log_t *log)
+{
+ return 0;
+}
+
#elif (NGX_HAVE_SCHED_SETAFFINITY)

void
@@ -50,4 +56,16 @@
}
}

+ngx_int_t
+ngx_getaffinity(ngx_cpuset_t *cpu_affinity, ngx_log_t *log)
+{
+ if (sched_getaffinity(0, sizeof(ngx_cpuset_t), cpu_affinity) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
+ "sched_getaffinity() failed");
+ return NGX_ERROR;
+ }
+
+ return CPU_COUNT(cpu_affinity);
+}
+
#endif
diff -r 7c614ef3c6ea -r 89793df28d1b src/os/unix/ngx_setaffinity.h
--- a/src/os/unix/ngx_setaffinity.h Wed Apr 18 16:11:41 2018 +0300
+++ b/src/os/unix/ngx_setaffinity.h Wed Apr 25 00:05:05 2018 +0800
@@ -23,12 +23,15 @@

#endif

+ngx_int_t ngx_getaffinity(ngx_cpuset_t *cpu_affinity, ngx_log_t *log);
void ngx_setaffinity(ngx_cpuset_t *cpu_affinity, ngx_log_t *log);

#else

#define ngx_setaffinity(cpu_affinity, log)

+#define ngx_getaffinity(cpu_affinity, log)
+
typedef uint64_t ngx_cpuset_t;

#endif_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel