Hello,
Thanks for your time !
I'm running PHP-FPM for the API I created that dynamicaly create Nginx vhosts (that's why I need to reload Nginx quite often).
It's not really use, like < 1000 requests / day.
99.999% of the trafic is just is associated with the 15K domain and Nginx just do a proxy_pass to a backend server.
Nginx listen to 100+ IPs (if that's important).
For the GZIP module, it's just a guest, for me it's logical that GZIP will use a lot of CPU but I thought that it could use some memory too for the buffering or some sort of cache.
I'm aware that it's unlikely the problem.
I used to have the ssl_session_cache on each vhosts but I moved it to the http level.
The memory consuption haven't change.
But, I have some limit_conn in each vhosts (included) so I think you're onto something with this !
Bellow the config I can share, with some comment :
### Begin of nginx.conf ###
user nginx;
worker_processes 6; # Doing some test on this, previous value was 8, also tested auto
worker_rlimit_nofile 65535;
error_log /var/log/nginx/err.crit error;
pid /run/nginx.pid;
events {
worker_connections 16192;
multi_accept on;
accept_mutex off;
use epoll;
}
http {
# I define a few geo config and maps too (~10)
# Used to block bad UA / domain / bot and apply rate limits
geo $whitelist {
default 0;
x.x.x.x 1; # a few IP range
}
map $whitelist $map_host {
0 $host;
1 "";
}
etag off;
if_modified_since off;
# I have a few rate limit too ~6-7 based on differents things
limit_conn_zone $map_perip zone=perip:1m;
limit_req_zone $map_perip zone=global1:1m rate=250r/s;
limit_req zone=global1 burst=300;
limit_req_status 429;
server_names_hash_bucket_size 512;
log_format upstreamlog '[$time_local] $remote_addr - $request - $server_name TO $upstream_addr : status=$upstream_status - response_time=$upstream_response_time - msec=$msec - request_time=$request_time';
log_format custom_server '$host - $server_name - $remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent $request_length $bytes_sent $request_time $server_port '
'"$http_referer" "$http_user_agent" ';
# debug err
map $status $loggable {
~^[234] 0;
default 1;
}
map $upstream_status $upstream_loggable {
~^[234] 0;
default 1;
}
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 30;
types_hash_max_size 2048;
client_max_body_size 256M;
aio threads;
include /etc/nginx/mime.types;
default_type application/octet-stream;
gzip on;
gzip_vary on;
gzip_disable "MSIE [1-6]\.";
gzip_proxied any;
gzip_http_version 1.0;
gzip_min_length 1000;
gzip_comp_level 1; # Doing some test, initial value was 9;
gzip_buffers 32 4k; # Doing some test, initial value was 64 8k;
gzip_types text/plain text/xml text/css application/x-javascript application/xml application/javascript application/xml+rss text/javascript application/atom+xml application/xhtml+xml image/svg+xml application/x-font-woff application/font-woff2;
brotli on;
brotli_static on;
brotli_types text/plain text/xml text/css application/x-javascript application/xml application/javascript application/xml+rss text/javascript application/atom+xml application/xhtml+xml image/svg+xml application/x-font-woff application/font-woff2;
brotli_comp_level 9;
ssl_session_cache shared:SSLCACHE:1m; # Doing some test, initial value was 10m and was defined in each vhosts
server {
listen x.x.x.x:80;
# + ~100 listen for ~100 ips
listen 80 default_server;
listen [::]:80 default_server;
server_name _;
root /opt/web/default;
location /stub_status {
stub_status;
allow 127.0.0.1;
deny all;
}
}
server {
listen x.x.x.x:443 ssl http2;
# + ~100 listen for ~100 ips
listen 443 ssl http2 default_server;
listen [::]:443 ssl http2 default_server;
server_name _;
root /opt/web/default;
ssl_certificate "/etc/pki/nginx/nginx.crt";
ssl_certificate_key "/etc/pki/nginx/private/nginx.key";
}
server {
listen x.x.x.x:445 ssl http2;
# + ~100 listen for ~100 ips
listen 445 ssl http2 default_server;
listen [::]:445 ssl http2 default_server;
server_name _;
root /opt/web/nginx/public;
ssl_certificate "/etc/pki/nginx/nginx.crt";
ssl_certificate_key "/etc/pki/nginx/private/nginx.key";
access_log /var/log/nginx/api_access_log custom_server;
error_log /var/log/nginx/api_error_log warn;
location / {
allow x.x.x.x/21;
deny all;
try_files $uri /index.php$is_args$args;
}
location ~ \.php$ {
try_files $uri =404;
fastcgi_index index.php;
fastcgi_pass unix:/var/run/php-fpm-nginx.sock;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
include /etc/nginx/fastcgi_params;
}
}
include /etc/nginx/vhosts/*.conf;
proxy_buffering on;
proxy_buffers 256 16k;
proxy_buffer_size 128k;
proxy_busy_buffers_size 256k;
proxy_temp_file_write_size 256k;
proxy_connect_timeout 300s;
proxy_read_timeout 300s;
proxy_send_timeout 300s;
open_file_cache max=2000 inactive=5m;
open_file_cache_valid 2m;
open_file_cache_min_uses 2;
open_file_cache_errors on;
}
### End of nginx.conf ###
### Begin of a typical vhosts ###
server {
listen x.x.x.x:443 ssl http2;
listen x.x.x.x:4430 ssl http2;
server_name xxxx.com www.xxxx.com;
ssl_certificate /etc/nginx/ssl/xxxx.eu.crt;
ssl_certificate_key /etc/nginx/ssl/xxxx.eu.key;
ssl_protocols TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
#ssl_session_cache shared:SSL:10m; # cf. nginx.conf
ssl_dhparam /etc/nginx/ssl/dhparam.pem;
ssl_session_timeout 5m;
proxy_intercept_errors on;
recursive_error_pages on;
location / {
proxy_intercept_errors on;
proxy_pass https://z.z.z.z:4430;
proxy_ssl_verify off;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Forwarded-Port 443;
proxy_set_header X-SSL yes;
include security-ssl.inc;
error_page 502 503 = @fallback1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Custom-IP yes;
}
# Fallback 1, in case of 502/503, retry the proxy_pass
location @fallback1 {
proxy_intercept_errors off;
proxy_pass https://z.z.z.z:4430;
proxy_ssl_verify off;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Forwarded-Port 443;
proxy_set_header X-SSL yes;
include security-ssl.inc;
proxy_set_header X-Retry 1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Custom-IP yes;
}
}
### End of vhosts ###
### security inc ###
open_log_file_cache max=1000 inactive=20s valid=1m min_uses=2;
access_log /var/log/nginx/web custom_server;
# Debug
#access_log /var/log/nginx/vhosts_5xx custom_server if=$loggable;
#access_log /var/log/nginx/vhosts_upstream_5xx upstreamlog if=$upstream_loggable;
limit_rate 4096k;
limit_conn perip 15;
limit_conn max_conn_to_vhost 150;
if ($limit_forbidbot) { return 444; }
if ($abusehost) { return 403; }
if ($abuseuri) { return 403; }
if ($request_method !~ ^(GET|PURGE|HEAD|OPTIONS|POST|PUT|DELETE)$ ) { return 444; }
### end of security inc ###
Thanks,
Alexis
