Merge branch 'master' into refresh-tor-build

lib.d/do-configure.pl

@@ -589,6 +589,7 @@ my @set_blank = qw(
     host_blacklist_re
     host_whitelist
     host_whitelist_re
+    log_separate
     nginx_modules_dirs
     no_cache_content_type
     no_cache_host

lib.d/generate-bw-code.pl

@@ -1,8 +1,8 @@
 #!/usr/bin/perl
 
 $warning = "(generated)";
-$begin = "# ---- BEGIN GENERATED CODE ---- -*- awk -*-\n\n";
-$end = "# ---- END GENERATED CODE ----\n";
+$begin = "# ---- BEGIN CODE GENERATED BY $0 ---- -*- awk -*-\n\n";
+$end   = "# ---- END CODE GENERATED BY $0 ----\n";
 
 $indent = "  ";
 @polite = ();
@@ -11,6 +11,7 @@ $indent = "  ";
 @white = ();
 @tail = ();
 
+$dont_onion = "set \$dont_onionify_response_headers 1; # dest URL must not be rewritten, prevent loops; cookies may migrate.";
 
 sub blackwhite {
     my ($operator, $lc_what, $a, $b) = @_;
@@ -88,9 +89,12 @@ while (<DATA>) {
     elsif ($how eq "redirect") {
         my $uc_what = uc($lc_what);
         push(@redirect, "%%IF %$uc_what%\n");
-        push(@redirect, "# redirect $lc_what: 1=regexp,2=code,3=dest $warning\n");
+        push(@redirect, "# redirect $lc_what: 1=regexp,2=code,3=dest (REQUEST_URI will be appended) $warning\n");
         push(@redirect, "%%CSV %$uc_what%\n");
-        push(@redirect, "$condition { return %2% %3%\$request_uri; }\n");
+        push(@redirect, "$condition {\n");
+        push(@redirect, "  $dont_onion\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge
+        push(@redirect, "  return %2% %3%\$request_uri;\n");
+        push(@redirect, "}\n");
         push(@redirect, "%%ENDCSV\n");
         push(@redirect, "%%ELSE\n");
         push(@redirect, "# no $lc_what\n");
@@ -99,20 +103,12 @@ while (<DATA>) {
     elsif ($how eq "fixed-redirect") {
         my $uc_what = uc($lc_what);
         push(@redirect, "%%IF %$uc_what%\n");
-        push(@redirect, "# fixed_redirect $lc_what: 1=regexp,2=code,3=dest (NO REQUEST_URI APPENDED) $warning\n");
+        push(@redirect, "# fixed_redirect $lc_what: 1=regexp,2=code,3=dest (REQUEST_URI will NOT be appended) $warning\n");
         push(@redirect, "%%CSV %$uc_what%\n");
-        push(@redirect, "$condition { return %2% %3%; }\n");
-        push(@redirect, "%%ENDCSV\n");
-        push(@redirect, "%%ELSE\n");
-        push(@redirect, "# no $lc_what\n");
-        push(@redirect, "%%ENDIF\n");
-    }
-    elsif ($how eq "legacy-redirect") {
-        my $uc_what = uc($lc_what);
-        push(@redirect, "%%IF %$uc_what%\n");
-        push(@redirect, "# legacy-redirect $lc_what: 1=regexp,2=dest,3=code $warning\n");
-        push(@redirect, "%%CSV %$uc_what%\n");
-        push(@redirect, "$condition { return %3% %2%\$request_uri; }\n");
+        push(@redirect, "$condition {\n");
+        push(@redirect, "  $dont_onion\n") if ($uc_what =~ /_HOST/); # this is a horrible kludge
+        push(@redirect, "  return %2% %3%;\n");
+        push(@redirect, "}\n");
         push(@redirect, "%%ENDCSV\n");
         push(@redirect, "%%ELSE\n");
         push(@redirect, "# no $lc_what\n");
@@ -198,16 +194,17 @@ block block_host if ( $http_host = "%0%" )
 block block_host_re if ( $http_host ~* "%0%" )
 block block_path if ( $uri = "%0%" )
 block block_path_re if ( $uri ~* "%0%" )
-## legacy
 block block_location location %0%
 block block_location_re location ~* "%0%"
-## query parameters
+
+# blocks on query parameters
 block block_param if ( $arg_%1% = "%2%" )
 block block_param_re if ( $arg_%1% ~* "%2%" )
 
 # redirects preserving the request_uri path
 redirect redirect_host if ( $host ~* "%1%" )
 redirect redirect_path if ( $uri ~* "%1%" )
+
 # redirects to a fixed url/path
 fixed-redirect redirect_fixed_host if ( $host ~* "%1%" )
 fixed-redirect redirect_fixed_path if ( $uri ~* "%1%" )

lib.d/lint.pl

@@ -53,6 +53,7 @@ my %known =
      'IS_SOFTMAP' => 1,
      'LEFT_TLD_RE' => 1,
      'LOG_DIR' => 1, # where logs for the current project live
+     'LOG_SEPARATE' => 1,
      'NGINX_ACTION_ABORT' => 1,
      'NGINX_BLOCK_BUSY_SIZE' => 1,
      'NGINX_BLOCK_COUNT' => 1,

templates.d/nginx-generated-blocks.conf

@@ -1,4 +1,4 @@
-    # ---- BEGIN GENERATED CODE ---- -*- awk -*-
+    # ---- BEGIN CODE GENERATED BY ../lib.d/generate-bw-code.pl ---- -*- awk -*-
 
     # blacklists (generated)
     %%IF %USER_AGENT_BLACKLIST%
@@ -222,33 +222,43 @@
 
     # redirects (generated)
     %%IF %REDIRECT_HOST%
-    # redirect redirect_host: 1=regexp,2=code,3=dest (generated)
+    # redirect redirect_host: 1=regexp,2=code,3=dest (REQUEST_URI will be appended) (generated)
     %%CSV %REDIRECT_HOST%
-    if ( $host ~* "%1%" ) { return %2% %3%$request_uri; }
+    if ( $host ~* "%1%" ) {
+      set $dont_onionify_response_headers 1; # dest URL must not be rewritten, prevent loops; cookies may migrate.
+      return %2% %3%$request_uri;
+    }
     %%ENDCSV
     %%ELSE
     # no redirect_host
     %%ENDIF
     %%IF %REDIRECT_PATH%
-    # redirect redirect_path: 1=regexp,2=code,3=dest (generated)
+    # redirect redirect_path: 1=regexp,2=code,3=dest (REQUEST_URI will be appended) (generated)
     %%CSV %REDIRECT_PATH%
-    if ( $uri ~* "%1%" ) { return %2% %3%$request_uri; }
+    if ( $uri ~* "%1%" ) {
+      return %2% %3%$request_uri;
+    }
     %%ENDCSV
     %%ELSE
     # no redirect_path
     %%ENDIF
     %%IF %REDIRECT_FIXED_HOST%
-    # fixed_redirect redirect_fixed_host: 1=regexp,2=code,3=dest (NO REQUEST_URI APPENDED) (generated)
+    # fixed_redirect redirect_fixed_host: 1=regexp,2=code,3=dest (REQUEST_URI will NOT be appended) (generated)
     %%CSV %REDIRECT_FIXED_HOST%
-    if ( $host ~* "%1%" ) { return %2% %3%; }
+    if ( $host ~* "%1%" ) {
+      set $dont_onionify_response_headers 1; # dest URL must not be rewritten, prevent loops; cookies may migrate.
+      return %2% %3%;
+    }
     %%ENDCSV
     %%ELSE
     # no redirect_fixed_host
     %%ENDIF
     %%IF %REDIRECT_FIXED_PATH%
-    # fixed_redirect redirect_fixed_path: 1=regexp,2=code,3=dest (NO REQUEST_URI APPENDED) (generated)
+    # fixed_redirect redirect_fixed_path: 1=regexp,2=code,3=dest (REQUEST_URI will NOT be appended) (generated)
     %%CSV %REDIRECT_FIXED_PATH%
-    if ( $uri ~* "%1%" ) { return %2% %3%; }
+    if ( $uri ~* "%1%" ) {
+      return %2% %3%;
+    }
     %%ENDCSV
     %%ELSE
     # no redirect_fixed_path
@@ -364,4 +374,4 @@
     # no param_whitelist_re
     %%ENDIF
 
-    # ---- END GENERATED CODE ----
+    # ---- END CODE GENERATED BY ../lib.d/generate-bw-code.pl ----

templates.d/nginx-generated-checks.conf

@@ -1,4 +1,4 @@
-      # ---- BEGIN GENERATED CODE ---- -*- awk -*-
+      # ---- BEGIN CODE GENERATED BY ../lib.d/generate-bw-code.pl ---- -*- awk -*-
 
       # whitelist checks (generated)
       %%IF %USER_AGENT_WHITELIST%
@@ -74,4 +74,4 @@
       # no param_whitelist_re
       %%ENDIF
 
-      # ---- END GENERATED CODE ----
+      # ---- END CODE GENERATED BY ../lib.d/generate-bw-code.pl ----

templates.d/nginx.conf.txt

@@ -101,7 +101,7 @@ http {
   # nginx caching disabled
   %%ENDIF
 
-  # logs
+  # logs (default)
   access_log %LOG_DIR%/nginx-access.log;
 
   # global settings
@@ -348,11 +348,23 @@ http {
       "Timing-Allow-Origin"
     }
 
-    local i, k
-    for i, k in ipairs(origin_rewrites) do
-      local v = ngx.header[k]
-      if v then
-        ngx.header[k] = dns_to_onion(v)
+    -- if REDIRECT_HOST (etc) is active we will set[1] $dont_onionify_response_headers
+    -- to `1`, which Lua will import[2] as string-or-nil and which requires manual[3]
+    -- type-conversion for comparison; simple not-nil tests are empirically prone to
+    -- misbehave, so it seems safest to simply test for an explicit value.
+    -- [1] see `generate-bw-code.pl` in two places
+    -- [2] https://github.com/openresty/lua-nginx-module#ngxvarvariable
+    -- [3] https://developer.roblox.com/en-us/articles/Type-Coercion-in-Lua#during-comparisons
+
+    -- it would be nice to reduce indentation and do a fastpath-return on `... == 1`
+    -- but that might break future work, below...
+    if tonumber(ngx.var.dont_onionify_response_headers) ~= 1 then
+      local i, k
+      for i, k in ipairs(origin_rewrites) do
+        local v = ngx.header[k]
+        if v then
+          ngx.header[k] = dns_to_onion(v)
+        end
       end
     end
 
@@ -504,6 +516,12 @@ http {
     listen unix:%PROJECT_DIR%/%ONION_DIRNAME%/port-80.sock;
     %%ENDIF
 
+    %%IF %LOG_SEPARATE%
+    access_log %LOG_DIR%/nginx-rdr443-%DNS_DOMAIN%.log;
+    %%ELSE
+    access_log %LOG_DIR%/nginx-rdr443.log;
+    %%ENDIF
+
     # subdomain regexp captures trailing dot, use carefully; does not need "~*"
     # NB: this regexp should be kept in-sync with the other FORCE_HTTPS copy
     server_name
@@ -550,6 +568,10 @@ http {
     listen unix:%PROJECT_DIR%/%ONION_DIRNAME%/port-443.sock ssl;
     %%ENDIF
 
+    %%IF %LOG_SEPARATE%
+    access_log %LOG_DIR%/nginx-access-%DNS_DOMAIN%.log;
+    %%ENDIF
+
     # subdomain regexp captures trailing dot, use carefully; does not need "~*"
     # NB: this regexp should be kept in-sync with the other FORCE_HTTPS copy
     server_name