April 10, 2020 11:26AM
Hi everyone,

We are using nginx as reverse proxy to cache static content for a moodle LMS site. The caching part is working fine but we started facing the cache path disk full issues. When we checked using du, it was reporting very low used space.Then we checked lsof output we found too many deleted file entries. We realized, that file gets deleted but it holds the FD and disk space is never released until we restart the nginx server. Most of the files were from proxy_temp_path location. We tried this with tmpfs and normal disk but the end result was same.

# nginx -v
nginx version: nginx/1.10.3 (Ubuntu)

# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 16.04.6 LTS
Release: 16.04
Codename: xenial

=================== df output =================
# df -h /mnt/
Filesystem Size Used Avail Use% Mounted on
/dev/sdb1 252G 8.7G 231G 4% /mnt

# du -sh /mnt/nginx/
4.0K /mnt/nginx/

================== nginx.conf ==================
user www-data;
worker_processes auto;
pid /run/nginx.pid;

events {
worker_connections 1024;
# multi_accept on;
}

http {

##
# Basic Settings
##

sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 0;
types_hash_max_size 2048;
client_max_body_size 2048M;
# server_tokens off;

# server_names_hash_bucket_size 64;
# server_name_in_redirect off;

include /etc/nginx/mime.types;
default_type application/octet-stream;

##
# SSL Settings
##

ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;

##
# Logging Settings
##

access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;

##
# Gzip Settings
##

gzip on;
gzip_disable "msie6";

# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;

##
# Virtual Host Configs
##

include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}

==================== SSL site config =====================
log_format rt_cache '$remote_addr - $upstream_cache_status [$time_local] '
'"$request" $status "$sent_http_content_type" $sent_http_content_encoding $body_bytes_sent '
'"$http_referer" "$http_user_agent"';
proxy_cache_path /var/cache/tmpfs levels=1:2 keys_zone=my_cache:1000m max_size=18g inactive=1d use_temp_path=off;;
proxy_temp_path /mnt/nginx;

server {
listen 443;
server_name xxx.yyy.com;
ssl_certificate /etc/apache2/ssl/xxx.yyy.com-nginx.crt;
ssl_certificate_key /etc/apache2/ssl/xxx.yyy.com.key;

ssl on;
ssl_session_cache builtin:1000 shared:SSL:10m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;
ssl_prefer_server_ciphers on;
access_log /var/log/nginx/xxx.yyy.com-access.log rt_cache;
#access_log off;
proxy_buffer_size 512k;
proxy_buffers 4 512k;
proxy_headers_hash_max_size 1024;

location /nginx-status {
stub_status on;
allow all;
}

location ~* \.(?:ico|jpg|css|png|js|swf|woff|eot|svg|ttf|html|gif|jpeg)$ {
aio threads;
proxy_cache my_cache;
add_header X-Proxy-Cache $upstream_cache_status;
expires 7d;
proxy_ignore_headers Cache-Control;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://localhost;
proxy_set_header Connection "";
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
proxy_read_timeout 600s;
send_timeout 600s;
proxy_http_version 1.1;
}

location / {
#proxy_buffering off;
proxy_ignore_headers Set-Cookie;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://localhost;
proxy_set_header Connection "";
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
proxy_read_timeout 600s;
send_timeout 600s;
proxy_http_version 1.1;
}
}
Subject Author Posted

Too many deleted open files in proxy_temp_path

patademahesh April 10, 2020 11:26AM

Re: Too many deleted open files in proxy_temp_path

J.R. April 10, 2020 11:20PM

Re: Too many deleted open files in proxy_temp_path

patademahesh April 11, 2020 01:01AM

Re: Too many deleted open files in proxy_temp_path

Maxim Dounin April 12, 2020 10:50AM

Re: Too many deleted open files in proxy_temp_path

renoirb April 12, 2020 11:39PM



Sorry, only registered users may post in this forum.

Click here to login

Online Users

Guests: 65
Record Number of Users: 6 on February 13, 2018
Record Number of Guests: 421 on December 02, 2018
Powered by nginx      Powered by FreeBSD      PHP Powered      Powered by MariaDB      ipv6 ready