Hi everyone,

We are using nginx as reverse proxy to cache static content for a moodle LMS site. The caching part is working fine but we started facing the cache path disk full issues. When we checked using du, it was reporting very low used space.Then we checked lsof output we found too many deleted file entries. We realized, that file gets deleted but it holds the FD and disk space is never released until we restart the nginx server. Most of the files were from proxy_temp_path location. We tried this with tmpfs and normal disk but the end result was same.

# nginx -v
nginx version: nginx/1.10.3 (Ubuntu)

# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 16.04.6 LTS
Release: 16.04
Codename: xenial

=================== df output =================
# df -h /mnt/
Filesystem Size Used Avail Use% Mounted on
/dev/sdb1 252G 8.7G 231G 4% /mnt

# du -sh /mnt/nginx/
4.0K /mnt/nginx/

================== nginx.conf ==================
user www-data;
worker_processes auto;
pid /run/nginx.pid;

events {
worker_connections 1024;
# multi_accept on;
}

http {

##
# Basic Settings
##

sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 0;
types_hash_max_size 2048;
client_max_body_size 2048M;
# server_tokens off;

# server_names_hash_bucket_size 64;
# server_name_in_redirect off;

include /etc/nginx/mime.types;
default_type application/octet-stream;

##
# SSL Settings
##

ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;

##
# Logging Settings
##

access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;

##
# Gzip Settings
##

gzip on;
gzip_disable "msie6";

# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;

##
# Virtual Host Configs
##

include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}

==================== SSL site config =====================
log_format rt_cache '$remote_addr - $upstream_cache_status [$time_local] '
'"$request" $status "$sent_http_content_type" $sent_http_content_encoding $body_bytes_sent '
'"$http_referer" "$http_user_agent"';
proxy_cache_path /var/cache/tmpfs levels=1:2 keys_zone=my_cache:1000m max_size=18g inactive=1d use_temp_path=off;;
proxy_temp_path /mnt/nginx;

server {
listen 443;
server_name xxx.yyy.com;
ssl_certificate /etc/apache2/ssl/xxx.yyy.com-nginx.crt;
ssl_certificate_key /etc/apache2/ssl/xxx.yyy.com.key;

ssl on;
ssl_session_cache builtin:1000 shared:SSL:10m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;
ssl_prefer_server_ciphers on;
access_log /var/log/nginx/xxx.yyy.com-access.log rt_cache;
#access_log off;
proxy_buffer_size 512k;
proxy_buffers 4 512k;
proxy_headers_hash_max_size 1024;

location /nginx-status {
stub_status on;
allow all;
}

location ~* \.(?:ico|jpg|css|png|js|swf|woff|eot|svg|ttf|html|gif|jpeg)$ {
aio threads;
proxy_cache my_cache;
add_header X-Proxy-Cache $upstream_cache_status;
expires 7d;
proxy_ignore_headers Cache-Control;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://localhost;
proxy_set_header Connection "";
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
proxy_read_timeout 600s;
send_timeout 600s;
proxy_http_version 1.1;
}

location / {
#proxy_buffering off;
proxy_ignore_headers Set-Cookie;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://localhost;
proxy_set_header Connection "";
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
proxy_read_timeout 600s;
send_timeout 600s;
proxy_http_version 1.1;
}
}