Merge pull request #67 from alecmuffett/20200605-inject-synthetic-headers

injectors for synthetic headers
2020-06-09 16:19:26 +01:00 · 2020-06-09 16:19:26 +01:00 · f121bc4c74
commit f121bc4c74
--- a/lib.d/do-configure.pl
+++ b/lib.d/do-configure.pl
@ -589,6 +589,8 @@ my @set_blank = qw(
    host_blacklist_re
    host_whitelist
    host_whitelist_re
+    inject_origin
+    inject_referer
    log_separate
    nginx_modules_dirs
    no_cache_content_type
--- a/lib.d/lint.pl
+++ b/lib.d/lint.pl
@ -50,6 +50,8 @@ my %known =
     'HOST_BLACKLIST_RE' => 1,
     'HOST_WHITELIST' => 1,
     'HOST_WHITELIST_RE' => 1,
+     'INJECT_ORIGIN' => 1,
+     'INJECT_REFERER' => 1,
     'IS_SOFTMAP' => 1,
     'LEFT_TLD_RE' => 1,
     'LOG_DIR' => 1, # where logs for the current project live
--- a/templates.d/nginx.conf.txt
+++ b/templates.d/nginx.conf.txt
@ -1,7 +1,21 @@
 # -*- awk -*-
 # EMACS awk mode works quite well for nginx configs

-# eotk (c) 2019 Alec Muffett
+# TODO LIST:
+
+# 1/ purge the TE request header being sent upstream; trailers and encodings are bad karma
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/TE
+
+# 2/ force the Transfer Encoding response header to `identity`?
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding
+
+# 3/ These...
+# X-Forwarded-Host request ?
+# X-XSS-Protection - response ?
+# X-DNS-Prefetch-Control - response security risk?
+# Via - request/response?
+
+# eotk (c) 2019-2020 Alec Muffett

 # SECURITY NOTE: the contents of this file, when actualised, should
 # not be made world-readable nor published without redaction;
@ -207,8 +221,8 @@ http {
  # no preserve subs (restore-phase)
  %%ENDIF

-  # o_to_d_lookup -> if cannot remap, return input.  note: old versions
-  # of lua-plugin cannot cope with code like o_to_d_mappings[o[1]]
+  # o2d_re_helper -> if cannot remap, return input. NB: old versions
+  # of lua-plugin cannot cope with code like o2d_mappings[o[1]]
  # because of `long bracket syntax`; the `[o[` freaks it out.
  # See: https://github.com/openresty/lua-nginx-module/issues/748
  init_by_lua_block {
@ -242,74 +256,115 @@ http {
      return string.sub(s, -string.len(x)) == x
    end

-    -- mapping onions to dns
+    -- useful shim for rewriting "scalar-or-table" values
+    ApplyReplacement = function (i, f)
+      if i == nil or i == "" then
+        return i
+      end
+      if (type(i) == "table") then
+        local k, v, result
+        result = {}
+        for k, v in ipairs(i) do
+          table.insert(result, ApplyReplacement(v, f)) -- recurse
+        end
+        return result
+      end
+      return f(i)
+    end

-    o_to_d_mappings = {}
+    -- MAPPING TABLES
+
+    o2d_mappings = {}
    %%BEGIN
-    o_to_d_mappings["%ONION_ADDRESS%"] = "%DNS_DOMAIN%"
+    o2d_mappings["%ONION_ADDRESS%"] = "%DNS_DOMAIN%"
    %%END

-    -- 1st element is the TLD boundary prefix, probably an empty string, maybe '2f'
-    -- 2nd element is the onion
-    o_to_d_lookup = function (m)
+    -- d2o_mappings = {}
+    %%BEGIN
+    -- d2o_mappings["%DNS_DOMAIN%"] = "%ONION_ADDRESS%"
+    %%END
+
+    -- injected origins
+    origin_replacement = {}
+    %%IF %INJECT_ORIGIN%
+    %%CSV %INJECT_ORIGIN%
+    origin_replacement["%1%"] = "%2%"
+    %%ENDCSV
+    %%ELSE
+    -- no origin replacements
+    %%ENDIF
+
+    -- injected referers
+    referer_replacement = {}
+    %%IF %INJECT_REFERER%
+    %%CSV %INJECT_REFERER%
+    referer_replacement["%1%"] = "%2%"
+    %%ENDCSV
+    %%ELSE
+    -- no referer replacements
+    %%ENDIF
+
+    -- EDITING FUNCTIONS
+
+    -- 1st element is the LEFT_TLD_RE boundary prefix, probably an empty string, maybe '2f'
+    -- 2nd element is the onion address
+    o2d_re_helper = function (m)
      local prefix = m[1]
      local k = m[2]
-      local v = ( o_to_d_mappings[k] or k )
+      local v = (o2d_mappings[k] or k)
      if (prefix == "") then -- fast happy-path response
        return v
      end
      return prefix .. v
    end

-    onion_to_dns = function (i)
-      if i == nil or i == "" then
-        return i
-      end
-      if (type(i) == "table") then
-        local j, k, result
-        result = {}
-        for j, k in ipairs(i) do
-          table.insert(result, onion_to_dns(k))
-        end
-        return result
-      end
-      local o, num, errs = ngx.re.gsub(i, "(%LEFT_TLD_RE%)([a-z2-7]{16}(?:[a-z2-7]{40})?\\.onion)\\b", o_to_d_lookup, "io")
+    o2d_search_and_replace = function (i)
+      -- because onion addresses are matchable, this can be done in one pass...
+      local o, num, errs = ngx.re.gsub(i, "(%LEFT_TLD_RE%)([a-z2-7]{16}(?:[a-z2-7]{40})?\\.onion)\\b", o2d_re_helper, "io")
      if errs == nil and num == 0 then
        return i -- nothing was changed, so return the original
      end
      return o
    end

-    -- mapping dns to onions, for experimentation
-
-    d_to_o_mappings = {}
-    %%BEGIN
-    d_to_o_mappings["%DNS_DOMAIN%"] = "%ONION_ADDRESS%"
-    %%END
-
-    d_to_o_lookup = function (m)
-      local k = m[1] -- see note above re: array syntax
-      return ( d_to_o_mappings[k] or k )
-    end
-
-    dns_to_onion = function (i)
-      if i == nil or i == "" or i == "*" then
-        return i
-      end
-      if (type(i) == "table") then
-        local j, k, result
-        result = {}
-        for j, k in ipairs(i) do
-          table.insert(result, dns_to_onion(k))
-        end
-        return result
-      end
+    d2o_search_and_replace = function (i)
      local num, errs
+      -- do a brute-force list of substitutions, because no systematic pattern to match
      %%BEGIN
      i, num, errs = ngx.re.gsub(i, "(%LEFT_TLD_RE%)%DNS_DOMAIN_RE2%\\b", "${1}%ONION_ADDRESS%", "io")
      %%END
      return i
    end
+
+    -- SHIMS
+
+    -- shim for origin rewrite, permitting injection
+    rewrite_origin_o2d = function (i, ctx)
+      return origin_replacement[ctx] or ApplyReplacement(i, o2d_search_and_replace)
+    end
+
+    -- shim for referer rewrite, permitting injection
+    rewrite_referer_o2d = function (i, ctx)
+      return referer_replacement[ctx] or ApplyReplacement(i, o2d_search_and_replace)
+    end
+
+    -- shim for cookie rewrite, permitting injection
+    rewrite_cookie_o2d = function (i, ctx)
+      return ApplyReplacement(i, o2d_search_and_replace)
+    end
+
+    -- shim for arbitrary rewrites
+    rewrite_o2d = function (i)
+      return ApplyReplacement(i, o2d_search_and_replace)
+    end
+
+    -- shim for scalar rewrites
+    rewrite_header_d2o = function (i)
+      if i == "*" then -- other special cases are in `ApplyReplacement`
+        return i
+      end
+      return ApplyReplacement(i, d2o_search_and_replace)
+    end
  }

  # filter the response headers en-route back to the user
@ -333,15 +388,16 @@ http {
    end

    local k, v
-
-    local origin_rewrites = {
-      "Access-Control-Allow-Origin",
+    local response_rewrites = {
      %%IF %SUPPRESS_HEADER_CSP%
      -- CSP headers are suppressed via SUPPRESS_HEADER_CSP
      %%ELSE
      "Content-Security-Policy",
      "Content-Security-Policy-Report-Only",
      %%ENDIF
+      "Access-Control-Allow-Origin",
+      "Content-Location",
+      "Feature-Policy",
      "Link",
      "Location",
      "Set-Cookie",
@ -360,10 +416,10 @@ http {
    -- but that might break future work, below...
    if tonumber(ngx.var.dont_onionify_response_headers) ~= 1 then
      local i, k
-      for i, k in ipairs(origin_rewrites) do
+      for i, k in ipairs(response_rewrites) do
        local v = ngx.header[k]
        if v then
-          ngx.header[k] = dns_to_onion(v)
+          ngx.header[k] = rewrite_header_d2o(v)
        end
      end
    end
@ -392,7 +448,7 @@ http {
  # it got there.
  # config usage: set debug_trap foo\\.regex\\.tld [...]
  body_filter_by_lua_block {
-    -- change auxh to Content-Encoding or Set-Cookie or whatever, if needed
+    -- change `auxh` to "Content-Encoding" or "Set-Cookie" or whatever you want to log...
    local auxh = "Access-Control-Allow-Origin"

    local i = ngx.arg[1]
@ -531,7 +587,7 @@ http {

    %%IF %SUPPRESS_TOR2WEB%
    # suppress tor2web traffic; "let them use clearnet"
-    if ( $http_x_tor2web ) {
+    if ($http_x_tor2web) {
      return 403 "%BLOCK_ERR%";
    }
    %%ELSE
@ -640,14 +696,14 @@ http {

      %%IF %COOKIE_LOCK%
      # check for cookie-lock
-      if ( $cookie_eotk_lock != "%COOKIE_LOCK%" ) { %NGINX_ACTION_ABORT%; }
+      if ($cookie_eotk_lock != "%COOKIE_LOCK%") { %NGINX_ACTION_ABORT%; }
      %%ELSE
      # no cookie-lock checks
      %%ENDIF

      # deonionify the request_uri for forwarding (both path and args)
      set_by_lua_block $request_uri2 {
-        return onion_to_dns(ngx.var.request_uri)
+        return rewrite_o2d(ngx.var.request_uri)
      }

      %%IF %DEONIONIFY_POST_BODIES%
@ -656,7 +712,7 @@ http {
        if ngx.req.get_method() == "POST" then
          ngx.req.read_body()
          local old = ngx.req.get_body_data()
-          local new = onion_to_dns(old)
+          local new = rewrite_o2d(old)
          if new ~= old then
            ngx.req.set_body_data(new)
          end
@ -667,15 +723,15 @@ http {
      %%ENDIF

      # note use of both $scheme and the deonionified uri (both path and args)
-      set $new_url "$scheme://${servernamesubdomain}%DNS_DOMAIN%$request_uri2";
-      proxy_pass $new_url;
+      proxy_pass "$scheme://${servernamesubdomain}%DNS_DOMAIN%$request_uri2";
      proxy_http_version 1.1;

-      # a note on proxy_set_header, add_header, similar methods, etc;
+      # a note re: proxy_set_header, add_header, similar methods, etc;
      # if you override *any* header then you will lose the other
      # headers inherited from the parent contexts:
      # https://blog.g3rt.nl/nginx-add_header-pitfall.html

+      # request_rewrites and injections:
      proxy_set_header X-From-Onion %X_FROM_ONION_VALUE%;
      proxy_set_header Host "${servernamesubdomain}%DNS_DOMAIN%";
      proxy_set_header Accept-Encoding "identity";
@ -683,16 +739,25 @@ http {
      proxy_set_header Upgrade $http_upgrade; # SSL
      proxy_ssl_server_name on; # SSL

-      # rewrite request referer
-      set_by_lua_block $referer2 { return onion_to_dns(ngx.var.http_referer) }
-      proxy_set_header Referer $referer2;
+      # NB: it's very tempting to use `$http_host` / ngx.var.http_host
+      # (or similar per-request information) as the context for the
+      # call to `rewrite_origin_o2d` and its friends; my thinking at
+      # the moment is that that would be "too narrow" / "too easy to
+      # break" because wildcards/CDN-hosts would need to be matched,
+      # and that sort of thing.  Switching on the TLD of upstream
+      # currently seems cognitively easier to deal with; plus: Lua
+      # interns short strings, so it should be fast.

-      # rewrite request origin
-      set_by_lua_block $origin2 { return onion_to_dns(ngx.var.http_origin) }
+      # rewrite/inject request origin TODO
+      set_by_lua_block $origin2 { return rewrite_origin_o2d(ngx.var.http_origin, "%DNS_DOMAIN%") }
      proxy_set_header Origin $origin2;

+      # rewrite/inject request referer TODO
+      set_by_lua_block $referer2 { return rewrite_referer_o2d(ngx.var.http_referer, "%DNS_DOMAIN%") }
+      proxy_set_header Referer $referer2;
+
      # rewrite request cookies
-      set_by_lua_block $cookie2 { return onion_to_dns(ngx.var.http_cookie) }
+      set_by_lua_block $cookie2 { return rewrite_cookie_o2d(ngx.var.http_cookie, "%DNS_DOMAIN%") }
      proxy_set_header Cookie $cookie2;

      %%IF %SUPPRESS_METHODS_EXCEPT_GET%
@ -714,14 +779,16 @@ http {
  # origin headers not debugged
  %%ENDIF

-  # header purge
+  # header purge: TODO: THIS NEEDS REVIEW
  more_clear_headers "Age";
  more_clear_headers "Server";
+  more_clear_headers "SourceMap"; # rewrites will break SourceMaps
  more_clear_headers "Via";
  more_clear_headers "X-From-Nginx";
  more_clear_headers "X-NA";
  more_clear_headers "X-Powered-By";
  more_clear_headers "X-Request-Id";
  more_clear_headers "X-Runtime";
+  more_clear_headers "X-SourceMap"; # rewrites will break SourceMaps
  more_clear_headers "X-Varnish";
 }