Hi all

I am developing a proxy service which uses NGINX to reverse proxy, kind of CDN-like but very specific to our needs. During this. I have hit an issue which I *think* is a bug but wanted to ask in case anyone can point to a solution or some reading I can do. The problem I have is this:

$upstream_http_NAME and $sent_http_NAME variables seem to be unpopulated/blank at some points in my config when other embedded variables *are* populated. This is probably best illustrated via an example, so here's a simplified test case I created:

user nginx;
worker_processes auto;
worker_priority -15;
worker_rlimit_nofile 50000;

events {
# worker_connections benefits from a large value in that it reduces error counts
worker_connections 20000;
multi_accept on;
}

http {


# for dynamic upstreams
resolver 8.8.8.8;


# Default includes
include /etc/nginx/current/mime.types;
default_type application/octet-stream;
include /etc/nginx/current/proxy.conf;

# Tuning options - these are mainly quite GTM-specific
server_tokens off;
keepalive_requests 1024;
keepalive_timeout 120s 120s;
sendfile on;
tcp_nodelay on;
tcp_nopush on;
client_header_timeout 5s;
open_file_cache max=16384 inactive=600s;
open_file_cache_valid 600s;
open_file_cache_min_uses 0;
open_file_cache_errors on;
output_buffers 64 128k;

# NEW - AIO
aio on;
directio 512;



# For small files and heavy load, this gives ~5-6x greater throughput (avoids swamping workers with one request)
postpone_output 0;
reset_timedout_connection on;
send_timeout 3s;
sendfile_max_chunk 1m;
large_client_header_buffers 8 8k;
connection_pool_size 4096;

# client_body_buffer_size - Sets buffer size for reading client request body. In case the request body is larger than the buffer, the whole body or only its part is written to a temporary file
client_body_buffer_size 8k;
client_header_buffer_size 8k;

# client_max_body_size - Sets the maximum allowed size of the client request body, specified in the “Content-Length” request header field. If the size in a request exceeds the configured value, the 413 (Request Entity Too Large) error is returned to the client
client_max_body_size 8m;


# We need to increase the hash bucket size as the R53 names are long!
server_names_hash_bucket_size 128;

# Same for proxy_headers_hash_max_size and proxy_headers_hash_bucket_size
proxy_headers_hash_max_size 4096;
proxy_headers_hash_bucket_size 1024;

# Logging
# NOTE: $host may need to in fact be $hostname
log_format standard '"$remote_addr" "$time_iso8601" "$request_method" "$scheme" "$host" "$request_uri" "$server_protocol" "$status" "$bytes_sent" "$http_referer" "$http_user_agent" "$ssl_protocol" "$ssl_cipher" "$ssl_server_name" "$ssl_session_reused"';
access_log /var/log/nginx/main-access.log standard;
error_log /var/log/nginx/main-error.log warn;

recursive_error_pages on;




# GeoIP config
# This appears to need an absolute path, despite the docs suggesting it doesn't.
# Path is defined in bake script so changes to that will break this
# geoip_country /usr/local/GeoIP.dat;
geoip_city /var/lib/GeoIP/GeoLiteCity.dat;
#geoip_proxy 0.0.0.0/0;
#geoip_proxy_recursive on;


# Proxy global configuration
# NOTES:
# proxy_cache_path is an http scope (global) directive
# keys_zone=shared_cache:XXXXm; denotes the amount of RAM to allow for cache index, 1MB ~7k-8k cached objects - exceeding the number of cached objects possible due to index size results in LRU invocation
proxy_cache_path /mnt/gtm_cache_data levels=1:2 use_temp_path=on keys_zone=shared_cache:256m inactive=1440m;
proxy_temp_path /mnt/gtm_cache_temp;

# NGINX recommends HTTP 1.1 for keepalive, proxied conns. (which we use)
proxy_http_version 1.1;
# NGINX recommends clearing the connection request header for keepalive http 1.1 conns
proxy_set_header Connection "";

# Conditions under which we want to try then next (if there is one) upstream server in the list & timeouts
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_next_upstream_timeout 5s;
proxy_next_upstream_tries 3;


ssl_session_cache shared:global_ssl_cache:128m;

ssl_session_timeout 120m;
ssl_session_tickets on;
#ssl_session_ticket_key /etc/nginx/current/tls/session/tkt.key;




# TEST CASE:
# Set up a DNS or hosts file entry to point to your NGINX instance running this config
# Hit this with URL: https://<HOSTNAME>/response-headers?Content-Type=text%2Fplain%3B+charset%3DUTF-8&via=1.1%20httpbin3&tester=hello

# To try to check if we can work around the problem below (vars not existing (?) at time of use/need), we'll try to copy the var via a map
map $upstream_http_via $copy_map_upstream_http_via {
default $upstream_http_via;
}

upstream origin {
server httpbin.org:443 resolve;
}

# Generic/common server for listen port configs
server {

# TMP removing fastopen=None backlog=-1 as the directives don't work!
listen *:80 so_keepalive=120s:30s:20 default_server;
listen *:443 ssl http2 reuseport deferred so_keepalive=120s:30s:20 default_server;


# This cert & key will never actually be used but are needed to allow the :443 operation - without them the connection will be closed
ssl_certificate /etc/nginx/current/tls/certs/default.crt;
ssl_certificate_key /etc/nginx/current/tls/private/default.key;

ssl_protocols TLSv1 TLSv1.1 TLSv1.2;

location / {

# To try to check if we can work around the problem below (vars not existing (?) at time of use/need), we'll try to copy the var via a set
set $copy_set_upstream_http_via $upstream_http_via;


more_set_headers "SENT_HTTP_VIA: $sent_http_via";
more_set_headers "HTTP_VIA: $http_via";
more_set_headers "UPSTREAM_HTTP_VIA: $upstream_http_via";

# For ref, the upstream is sending e.g. "via: 1.1 string"

# Problem: These do not match, $upstream_http_via appears not to be populated at this point
# if ( $sent_http_via ~* "^[0-9]\.[0-9]\ .+$" ) {
# if ($upstream_http_via ~* "^[0-9]\.[0-9]\ .+") {
# if ($sent_http_via ~* "^[0-9]\.[0-9]\ .+") {
# if ($upstream_http_via) {

# This does match - for obvious reasons
if ($upstream_http_via ~* ".*") {

# Problem: $upstream_http_via appears not to be populated at this point...
set $via_comp "$upstream_http_via 1.y BLAH";
more_set_headers "IF_VIA_COMP: Value is $via_comp";

# Just to demo the string concat - we'll use a different embedded var
set $test_comp "$ssl_protocol BLAH";
more_set_headers "IF_TEST_COMP: Value is $test_comp";

# Does the map-copied var exist?
set $via_comp_copy_map "$copy_map_upstream_http_via 1.y BLAH";
more_set_headers "IF_VIA_COMP_COPY_MAP: Value is $via_comp_copy_map";

# Does the set-copied var exist?
set $via_comp_copy_set "$copy_set_upstream_http_via 1.y BLAH";
more_set_headers "IF_VIA_COMP_COPY_SET: Value is $via_comp_copy_set";

# Does a different $upstream_http_X var work? - NO
set $alt_comp "$upstream_http_tester 1.y BLAH";
more_set_headers "IF_ALT_COMP: Value is $alt_comp";

# ...but $upstream_http_via IS populated at this point
more_set_headers "UPSTREAM_HTTP_VIA_IF: $upstream_http_via";
more_set_headers "HTTP_VIA_IF: $http_via";
more_set_headers "SENT_HTTP_VIA_IF: $sent_http_via";
}

proxy_pass https://origin;
}

# END TEST CASE


}
}

(Sorry, couldn't figure out how to markup the config)

The response headers from a request to this NGIX instance is e.g.:

access-control-allow-credentials:true
access-control-allow-origin:*
content-length:161
content-type:text/plain; charset=UTF-8
date:Wed, 30 Mar 2016 10:31:55 GMT
if_alt_comp:Value is 1.y BLAH
if_test_comp:Value is TLSv1.2 BLAH
if_via_comp:Value is 1.y BLAH
if_via_comp_copy_map:Value is 1.y BLAH
if_via_comp_copy_set:Value is 1.y BLAH
sent_http_via:1.1 httpbin3
sent_http_via_if:1.1 httpbin3
server:nginx
status:200
tester:hello
upstream_http_via:1.1 httpbin3
upstream_http_via_if:1.1 httpbin3
via:1.1 httpbin3

So you can see from the section:
if_alt_comp:Value is 1.y BLAH
if_test_comp:Value is TLSv1.2 BLAH
if_via_comp:Value is 1.y BLAH
if_via_comp_copy_map:Value is 1.y BLAH
if_via_comp_copy_set:Value is 1.y BLAH

That there's a blank space where the value from $upstream_http_via or $sent_http_via should be.

So my questions are:
Can anyone see something I have done wrong?
Is this expected behaviour (if yes, why? Seems strange some vars behave differently)
Does anyone have a workaround or solution?

Many thanks in advance if anyone can offer any help.

Cheers
Neil