From df8532be103be823a7025dd071a0a40093677f82 Mon Sep 17 00:00:00 2001 From: Alec Muffett Date: Thu, 19 Mar 2020 12:38:31 +0000 Subject: [PATCH 1/7] commit: first cut, separate files for logs --- lib.d/do-configure.pl | 1 + lib.d/lint.pl | 1 + templates.d/nginx.conf.txt | 12 +++++++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib.d/do-configure.pl b/lib.d/do-configure.pl index aac3c12..d55133f 100755 --- a/lib.d/do-configure.pl +++ b/lib.d/do-configure.pl @@ -590,6 +590,7 @@ my @set_blank = qw( host_blacklist_re host_whitelist host_whitelist_re + log_separate nginx_modules_dirs no_cache_content_type no_cache_host diff --git a/lib.d/lint.pl b/lib.d/lint.pl index e32fc49..6e26904 100755 --- a/lib.d/lint.pl +++ b/lib.d/lint.pl @@ -53,6 +53,7 @@ my %known = 'IS_SOFTMAP' => 1, 'LEFT_TLD_RE' => 1, 'LOG_DIR' => 1, # where logs for the current project live + 'LOG_SEPARATES' => 1, 'NGINX_ACTION_ABORT' => 1, 'NGINX_BLOCK_BUSY_SIZE' => 1, 'NGINX_BLOCK_COUNT' => 1, diff --git a/templates.d/nginx.conf.txt b/templates.d/nginx.conf.txt index dce8f8e..cd4d9cc 100644 --- a/templates.d/nginx.conf.txt +++ b/templates.d/nginx.conf.txt @@ -101,7 +101,7 @@ http { # nginx caching disabled %%ENDIF - # logs + # logs (default) access_log %LOG_DIR%/nginx-access.log; # global settings @@ -504,6 +504,12 @@ http { listen unix:%PROJECT_DIR%/%ONION_DIRNAME%/port-80.sock; %%ENDIF + %%IF %LOG_SEPARATE% + access_log %LOG_DIR%/nginx-rdr443-%DNS_DOMAIN%.log; + %%ELSE + access_log %LOG_DIR%/nginx-rdr443.log; + %%ENDIF + # subdomain regexp captures trailing dot, use carefully; does not need "~*" # NB: this regexp should be kept in-sync with the other FORCE_HTTPS copy server_name @@ -550,6 +556,10 @@ http { listen unix:%PROJECT_DIR%/%ONION_DIRNAME%/port-443.sock ssl; %%ENDIF + %%IF %LOG_SEPARATE% + access_log %LOG_DIR%/nginx-access-%DNS_DOMAIN%.log; + %%ENDIF + # subdomain regexp captures trailing dot, use carefully; does not need "~*" # NB: this regexp should be kept in-sync with the other FORCE_HTTPS copy server_name From b03079673ff8773a9530a2065c6ffc7871340b41 Mon Sep 17 00:00:00 2001 From: Alec Muffett Date: Thu, 19 Mar 2020 16:18:54 +0000 Subject: [PATCH 2/7] commit: fix typo --- lib.d/lint.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib.d/lint.pl b/lib.d/lint.pl index 6e26904..00e26fb 100755 --- a/lib.d/lint.pl +++ b/lib.d/lint.pl @@ -53,7 +53,7 @@ my %known = 'IS_SOFTMAP' => 1, 'LEFT_TLD_RE' => 1, 'LOG_DIR' => 1, # where logs for the current project live - 'LOG_SEPARATES' => 1, + 'LOG_SEPARATE' => 1, 'NGINX_ACTION_ABORT' => 1, 'NGINX_BLOCK_BUSY_SIZE' => 1, 'NGINX_BLOCK_COUNT' => 1, From df704d43de4b68ad08ae2fdb44a922b7a9b82106 Mon Sep 17 00:00:00 2001 From: Alec Muffett Date: Thu, 4 Jun 2020 15:25:14 +0000 Subject: [PATCH 3/7] commit: first draft of hostname preservation --- lib.d/generate-bw-code.pl | 35 +++++++++++-------------- templates.d/nginx-generated-blocks.conf | 30 ++++++++++++++------- templates.d/nginx-generated-checks.conf | 4 +-- templates.d/nginx.conf.txt | 19 ++++++++++---- 4 files changed, 51 insertions(+), 37 deletions(-) diff --git a/lib.d/generate-bw-code.pl b/lib.d/generate-bw-code.pl index 5b43157..84ebd03 100755 --- a/lib.d/generate-bw-code.pl +++ b/lib.d/generate-bw-code.pl @@ -1,8 +1,8 @@ #!/usr/bin/perl $warning = "(generated)"; -$begin = "# ---- BEGIN GENERATED CODE ---- -*- awk -*-\n\n"; -$end = "# ---- END GENERATED CODE ----\n"; +$begin = "# ---- BEGIN CODE GENERATED BY $0 ---- -*- awk -*-\n\n"; +$end = "# ---- END CODE GENERATED BY $0 ----\n"; $indent = " "; @polite = (); @@ -11,7 +11,6 @@ $indent = " "; @white = (); @tail = (); - sub blackwhite { my ($operator, $lc_what, $a, $b) = @_; my $uc_what = uc($lc_what); @@ -88,9 +87,12 @@ while () { elsif ($how eq "redirect") { my $uc_what = uc($lc_what); push(@redirect, "%%IF %$uc_what%\n"); - push(@redirect, "# redirect $lc_what: 1=regexp,2=code,3=dest $warning\n"); + push(@redirect, "# redirect $lc_what: 1=regexp,2=code,3=dest (REQUEST_URI will be appended) $warning\n"); push(@redirect, "%%CSV %$uc_what%\n"); - push(@redirect, "$condition { return %2% %3%\$request_uri; }\n"); + push(@redirect, "$condition {\n"); + push(@redirect, " set \$onionify_response_headers 0;\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge + push(@redirect, " return %2% %3%\$request_uri;\n"); + push(@redirect, "}\n"); push(@redirect, "%%ENDCSV\n"); push(@redirect, "%%ELSE\n"); push(@redirect, "# no $lc_what\n"); @@ -99,20 +101,12 @@ while () { elsif ($how eq "fixed-redirect") { my $uc_what = uc($lc_what); push(@redirect, "%%IF %$uc_what%\n"); - push(@redirect, "# fixed_redirect $lc_what: 1=regexp,2=code,3=dest (NO REQUEST_URI APPENDED) $warning\n"); + push(@redirect, "# fixed_redirect $lc_what: 1=regexp,2=code,3=dest (REQUEST_URI will NOT be appended) $warning\n"); push(@redirect, "%%CSV %$uc_what%\n"); - push(@redirect, "$condition { return %2% %3%; }\n"); - push(@redirect, "%%ENDCSV\n"); - push(@redirect, "%%ELSE\n"); - push(@redirect, "# no $lc_what\n"); - push(@redirect, "%%ENDIF\n"); - } - elsif ($how eq "legacy-redirect") { - my $uc_what = uc($lc_what); - push(@redirect, "%%IF %$uc_what%\n"); - push(@redirect, "# legacy-redirect $lc_what: 1=regexp,2=dest,3=code $warning\n"); - push(@redirect, "%%CSV %$uc_what%\n"); - push(@redirect, "$condition { return %3% %2%\$request_uri; }\n"); + push(@redirect, "$condition {\n"); + push(@redirect, " set \$onionify_response_headers 0;\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge + push(@redirect, " return %2% %3%;\n"); + push(@redirect, "}\n"); push(@redirect, "%%ENDCSV\n"); push(@redirect, "%%ELSE\n"); push(@redirect, "# no $lc_what\n"); @@ -198,16 +192,17 @@ block block_host if ( $http_host = "%0%" ) block block_host_re if ( $http_host ~* "%0%" ) block block_path if ( $uri = "%0%" ) block block_path_re if ( $uri ~* "%0%" ) -## legacy block block_location location %0% block block_location_re location ~* "%0%" -## query parameters + +# blocks on query parameters block block_param if ( $arg_%1% = "%2%" ) block block_param_re if ( $arg_%1% ~* "%2%" ) # redirects preserving the request_uri path redirect redirect_host if ( $host ~* "%1%" ) redirect redirect_path if ( $uri ~* "%1%" ) + # redirects to a fixed url/path fixed-redirect redirect_fixed_host if ( $host ~* "%1%" ) fixed-redirect redirect_fixed_path if ( $uri ~* "%1%" ) diff --git a/templates.d/nginx-generated-blocks.conf b/templates.d/nginx-generated-blocks.conf index ae9b771..c747a09 100644 --- a/templates.d/nginx-generated-blocks.conf +++ b/templates.d/nginx-generated-blocks.conf @@ -1,4 +1,4 @@ - # ---- BEGIN GENERATED CODE ---- -*- awk -*- + # ---- BEGIN CODE GENERATED BY ../lib.d/generate-bw-code.pl ---- -*- awk -*- # blacklists (generated) %%IF %USER_AGENT_BLACKLIST% @@ -222,33 +222,43 @@ # redirects (generated) %%IF %REDIRECT_HOST% - # redirect redirect_host: 1=regexp,2=code,3=dest (generated) + # redirect redirect_host: 1=regexp,2=code,3=dest (REQUEST_URI will be appended) (generated) %%CSV %REDIRECT_HOST% - if ( $host ~* "%1%" ) { return %2% %3%$request_uri; } + if ( $host ~* "%1%" ) { + set $onionify_response_headers 0; + return %2% %3%$request_uri; + } %%ENDCSV %%ELSE # no redirect_host %%ENDIF %%IF %REDIRECT_PATH% - # redirect redirect_path: 1=regexp,2=code,3=dest (generated) + # redirect redirect_path: 1=regexp,2=code,3=dest (REQUEST_URI will be appended) (generated) %%CSV %REDIRECT_PATH% - if ( $uri ~* "%1%" ) { return %2% %3%$request_uri; } + if ( $uri ~* "%1%" ) { + return %2% %3%$request_uri; + } %%ENDCSV %%ELSE # no redirect_path %%ENDIF %%IF %REDIRECT_FIXED_HOST% - # fixed_redirect redirect_fixed_host: 1=regexp,2=code,3=dest (NO REQUEST_URI APPENDED) (generated) + # fixed_redirect redirect_fixed_host: 1=regexp,2=code,3=dest (REQUEST_URI will NOT be appended) (generated) %%CSV %REDIRECT_FIXED_HOST% - if ( $host ~* "%1%" ) { return %2% %3%; } + if ( $host ~* "%1%" ) { + set $onionify_response_headers 0; + return %2% %3%; + } %%ENDCSV %%ELSE # no redirect_fixed_host %%ENDIF %%IF %REDIRECT_FIXED_PATH% - # fixed_redirect redirect_fixed_path: 1=regexp,2=code,3=dest (NO REQUEST_URI APPENDED) (generated) + # fixed_redirect redirect_fixed_path: 1=regexp,2=code,3=dest (REQUEST_URI will NOT be appended) (generated) %%CSV %REDIRECT_FIXED_PATH% - if ( $uri ~* "%1%" ) { return %2% %3%; } + if ( $uri ~* "%1%" ) { + return %2% %3%; + } %%ENDCSV %%ELSE # no redirect_fixed_path @@ -364,4 +374,4 @@ # no param_whitelist_re %%ENDIF - # ---- END GENERATED CODE ---- + # ---- END CODE GENERATED BY ../lib.d/generate-bw-code.pl ---- diff --git a/templates.d/nginx-generated-checks.conf b/templates.d/nginx-generated-checks.conf index e9b7692..a8f2206 100644 --- a/templates.d/nginx-generated-checks.conf +++ b/templates.d/nginx-generated-checks.conf @@ -1,4 +1,4 @@ - # ---- BEGIN GENERATED CODE ---- -*- awk -*- + # ---- BEGIN CODE GENERATED BY ../lib.d/generate-bw-code.pl ---- -*- awk -*- # whitelist checks (generated) %%IF %USER_AGENT_WHITELIST% @@ -74,4 +74,4 @@ # no param_whitelist_re %%ENDIF - # ---- END GENERATED CODE ---- + # ---- END CODE GENERATED BY ../lib.d/generate-bw-code.pl ---- diff --git a/templates.d/nginx.conf.txt b/templates.d/nginx.conf.txt index cd4d9cc..79a3b12 100644 --- a/templates.d/nginx.conf.txt +++ b/templates.d/nginx.conf.txt @@ -348,11 +348,15 @@ http { "Timing-Allow-Origin" } - local i, k - for i, k in ipairs(origin_rewrites) do - local v = ngx.header[k] - if v then - ngx.header[k] = dns_to_onion(v) + -- shall we onionify the response headers? probably "yes" + -- but we need to doublecheck because of REDIRECT_HOST, etc... + if ngx.var.onionify_response_headers do + local i, k + for i, k in ipairs(origin_rewrites) do + local v = ngx.header[k] + if v then + ngx.header[k] = dns_to_onion(v) + end end end @@ -654,6 +658,11 @@ http { # no deonionify_post_bodies %%ENDIF + # onionify response headers; this should is default for 99%+ of + # requests, however you may in some circumstances want users to + # (eg:) escape or be redirected to the non-onionified site. + set $onionify_response_headers 1; + # note use of both $scheme and the deonionified uri (both path and args) set $new_url "$scheme://${servernamesubdomain}%DNS_DOMAIN%$request_uri2"; proxy_pass $new_url; From c3400eb7f844ef398809344e2403bcfe0f32eaf0 Mon Sep 17 00:00:00 2001 From: Alec Muffett Date: Thu, 4 Jun 2020 16:03:43 +0000 Subject: [PATCH 4/7] commit: nits --- lib.d/generate-bw-code.pl | 4 ++-- templates.d/nginx-generated-blocks.conf | 4 ++-- templates.d/nginx.conf.txt | 9 +++------ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib.d/generate-bw-code.pl b/lib.d/generate-bw-code.pl index 84ebd03..0d378a7 100755 --- a/lib.d/generate-bw-code.pl +++ b/lib.d/generate-bw-code.pl @@ -90,7 +90,7 @@ while () { push(@redirect, "# redirect $lc_what: 1=regexp,2=code,3=dest (REQUEST_URI will be appended) $warning\n"); push(@redirect, "%%CSV %$uc_what%\n"); push(@redirect, "$condition {\n"); - push(@redirect, " set \$onionify_response_headers 0;\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge + push(@redirect, " set \$dont_onionify_response_headers 1;\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge push(@redirect, " return %2% %3%\$request_uri;\n"); push(@redirect, "}\n"); push(@redirect, "%%ENDCSV\n"); @@ -104,7 +104,7 @@ while () { push(@redirect, "# fixed_redirect $lc_what: 1=regexp,2=code,3=dest (REQUEST_URI will NOT be appended) $warning\n"); push(@redirect, "%%CSV %$uc_what%\n"); push(@redirect, "$condition {\n"); - push(@redirect, " set \$onionify_response_headers 0;\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge + push(@redirect, " set \$dont_onionify_response_headers 1;\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge push(@redirect, " return %2% %3%;\n"); push(@redirect, "}\n"); push(@redirect, "%%ENDCSV\n"); diff --git a/templates.d/nginx-generated-blocks.conf b/templates.d/nginx-generated-blocks.conf index c747a09..c157686 100644 --- a/templates.d/nginx-generated-blocks.conf +++ b/templates.d/nginx-generated-blocks.conf @@ -225,7 +225,7 @@ # redirect redirect_host: 1=regexp,2=code,3=dest (REQUEST_URI will be appended) (generated) %%CSV %REDIRECT_HOST% if ( $host ~* "%1%" ) { - set $onionify_response_headers 0; + set $dont_onionify_response_headers 1; return %2% %3%$request_uri; } %%ENDCSV @@ -246,7 +246,7 @@ # fixed_redirect redirect_fixed_host: 1=regexp,2=code,3=dest (REQUEST_URI will NOT be appended) (generated) %%CSV %REDIRECT_FIXED_HOST% if ( $host ~* "%1%" ) { - set $onionify_response_headers 0; + set $dont_onionify_response_headers 1; return %2% %3%; } %%ENDCSV diff --git a/templates.d/nginx.conf.txt b/templates.d/nginx.conf.txt index 79a3b12..8248e18 100644 --- a/templates.d/nginx.conf.txt +++ b/templates.d/nginx.conf.txt @@ -350,7 +350,9 @@ http { -- shall we onionify the response headers? probably "yes" -- but we need to doublecheck because of REDIRECT_HOST, etc... - if ngx.var.onionify_response_headers do + -- "Undefined Nginx variables are evaluated to nil" + -- https://github.com/openresty/lua-nginx-module#ngxvarvariable + if not ngx.var.dont_onionify_response_headers then local i, k for i, k in ipairs(origin_rewrites) do local v = ngx.header[k] @@ -658,11 +660,6 @@ http { # no deonionify_post_bodies %%ENDIF - # onionify response headers; this should is default for 99%+ of - # requests, however you may in some circumstances want users to - # (eg:) escape or be redirected to the non-onionified site. - set $onionify_response_headers 1; - # note use of both $scheme and the deonionified uri (both path and args) set $new_url "$scheme://${servernamesubdomain}%DNS_DOMAIN%$request_uri2"; proxy_pass $new_url; From f2569d9ede39f3c26739c89b92788bc877c505d1 Mon Sep 17 00:00:00 2001 From: Alec Muffett Date: Thu, 4 Jun 2020 16:49:24 +0000 Subject: [PATCH 5/7] commit: coerce type of set flag to numeric to be explicit for comparison --- templates.d/nginx.conf.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates.d/nginx.conf.txt b/templates.d/nginx.conf.txt index 8248e18..005e311 100644 --- a/templates.d/nginx.conf.txt +++ b/templates.d/nginx.conf.txt @@ -352,7 +352,7 @@ http { -- but we need to doublecheck because of REDIRECT_HOST, etc... -- "Undefined Nginx variables are evaluated to nil" -- https://github.com/openresty/lua-nginx-module#ngxvarvariable - if not ngx.var.dont_onionify_response_headers then + if tonumber(ngx.var.dont_onionify_response_headers) ~= 1 then local i, k for i, k in ipairs(origin_rewrites) do local v = ngx.header[k] From e1f77730a67a16ea2c89949e27b3e7a3b303422d Mon Sep 17 00:00:00 2001 From: Alec Muffett Date: Fri, 5 Jun 2020 10:37:34 +0000 Subject: [PATCH 6/7] commit: slightly nicer generated code --- lib.d/generate-bw-code.pl | 6 ++++-- templates.d/nginx-generated-blocks.conf | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib.d/generate-bw-code.pl b/lib.d/generate-bw-code.pl index 0d378a7..fa5479f 100755 --- a/lib.d/generate-bw-code.pl +++ b/lib.d/generate-bw-code.pl @@ -11,6 +11,8 @@ $indent = " "; @white = (); @tail = (); +$dont_onion = "set \$dont_onionify_response_headers 1; # dest URL must not be rewritten, prevent loops; cookies may migrate."; + sub blackwhite { my ($operator, $lc_what, $a, $b) = @_; my $uc_what = uc($lc_what); @@ -90,7 +92,7 @@ while () { push(@redirect, "# redirect $lc_what: 1=regexp,2=code,3=dest (REQUEST_URI will be appended) $warning\n"); push(@redirect, "%%CSV %$uc_what%\n"); push(@redirect, "$condition {\n"); - push(@redirect, " set \$dont_onionify_response_headers 1;\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge + push(@redirect, " $dont_onion\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge push(@redirect, " return %2% %3%\$request_uri;\n"); push(@redirect, "}\n"); push(@redirect, "%%ENDCSV\n"); @@ -104,7 +106,7 @@ while () { push(@redirect, "# fixed_redirect $lc_what: 1=regexp,2=code,3=dest (REQUEST_URI will NOT be appended) $warning\n"); push(@redirect, "%%CSV %$uc_what%\n"); push(@redirect, "$condition {\n"); - push(@redirect, " set \$dont_onionify_response_headers 1;\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge + push(@redirect, " $dont_onion\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge push(@redirect, " return %2% %3%;\n"); push(@redirect, "}\n"); push(@redirect, "%%ENDCSV\n"); diff --git a/templates.d/nginx-generated-blocks.conf b/templates.d/nginx-generated-blocks.conf index c157686..c451c1f 100644 --- a/templates.d/nginx-generated-blocks.conf +++ b/templates.d/nginx-generated-blocks.conf @@ -225,7 +225,7 @@ # redirect redirect_host: 1=regexp,2=code,3=dest (REQUEST_URI will be appended) (generated) %%CSV %REDIRECT_HOST% if ( $host ~* "%1%" ) { - set $dont_onionify_response_headers 1; + set $dont_onionify_response_headers 1; # dest URL must not be rewritten, prevent loops; cookies may migrate. return %2% %3%$request_uri; } %%ENDCSV @@ -246,7 +246,7 @@ # fixed_redirect redirect_fixed_host: 1=regexp,2=code,3=dest (REQUEST_URI will NOT be appended) (generated) %%CSV %REDIRECT_FIXED_HOST% if ( $host ~* "%1%" ) { - set $dont_onionify_response_headers 1; + set $dont_onionify_response_headers 1; # dest URL must not be rewritten, prevent loops; cookies may migrate. return %2% %3%; } %%ENDCSV From 780a1623a79dfb44de0285a0fee779f915c329ac Mon Sep 17 00:00:00 2001 From: Alec Muffett Date: Fri, 5 Jun 2020 10:55:30 +0000 Subject: [PATCH 7/7] commit: better comment --- templates.d/nginx.conf.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/templates.d/nginx.conf.txt b/templates.d/nginx.conf.txt index 005e311..a4ffc48 100644 --- a/templates.d/nginx.conf.txt +++ b/templates.d/nginx.conf.txt @@ -348,10 +348,16 @@ http { "Timing-Allow-Origin" } - -- shall we onionify the response headers? probably "yes" - -- but we need to doublecheck because of REDIRECT_HOST, etc... - -- "Undefined Nginx variables are evaluated to nil" - -- https://github.com/openresty/lua-nginx-module#ngxvarvariable + -- if REDIRECT_HOST (etc) is active we will set[1] $dont_onionify_response_headers + -- to `1`, which Lua will import[2] as string-or-nil and which requires manual[3] + -- type-conversion for comparison; simple not-nil tests are empirically prone to + -- misbehave, so it seems safest to simply test for an explicit value. + -- [1] see `generate-bw-code.pl` in two places + -- [2] https://github.com/openresty/lua-nginx-module#ngxvarvariable + -- [3] https://developer.roblox.com/en-us/articles/Type-Coercion-in-Lua#during-comparisons + + -- it would be nice to reduce indentation and do a fastpath-return on `... == 1` + -- but that might break future work, below... if tonumber(ngx.var.dont_onionify_response_headers) ~= 1 then local i, k for i, k in ipairs(origin_rewrites) do