Get nginx to proxy stuff for us!

.gitignore

@@ -6,7 +6,7 @@
 .idea/*
 .venv
 .vscode
-/*.env
+/*.env*
 /build
 /cache/
 /docs/_build

docker/nginx.conf

@@ -15,6 +15,8 @@ http {
         server "127.0.0.1:8001";
     }
 
+    # access_log /dev/stdout;
+
     server {
         listen 8000;
         listen [::]:8000;
@@ -26,10 +28,7 @@ http {
         ignore_invalid_headers on;
         proxy_connect_timeout 900;
 
-        proxy_headers_hash_max_size 1024;
+        client_max_body_size 100M;
-        proxy_headers_hash_bucket_size 128;
-
-        client_max_body_size 512M;
         client_body_buffer_size 128k;
         charset utf-8;
 
@@ -38,26 +37,68 @@ http {
         proxy_http_version 1.1;
         proxy_cache takahe;
 
+        # Serves static files from the collected dir
         location /static/ {
             alias /takahe/static-collected/;
         }
 
+        # Proxies media and remote media with caching
         location ~* ^/(media|proxy) {
+            # Cache media and proxied resources
             proxy_cache_key $host$uri;
-            proxy_cache_valid 200 304 720h;
+            proxy_cache_valid 200 304 4h;
-            proxy_cache_valid 301 307 12h;
+            proxy_cache_valid 301 307 4h;
             proxy_cache_valid 500 502 503 504 0s;
-            proxy_cache_valid any 72h;
+            proxy_cache_valid any 1h;
-
             add_header X-Cache $upstream_cache_status;
 
+            # Signal to Takahē that we support full URI accel proxying
+            proxy_set_header X-Takahe-Accel true;
+
             proxy_pass http://takahe;
         }
 
+        # Internal target for X-Accel redirects that stashes the URI in a var
+        location /__takahe_accel__/ {
+            internal;
+            set $takahe_realuri $upstream_http_x_takahe_realuri;
+            rewrite ^/(.+) /__takahe_accel__/real/;
+        }
+
+        # Real internal-only target for X-Accel redirects
+        location /__takahe_accel__/real/ {
+            # Only allow internal redirects
+            internal;
+
+            # Reconstruct the remote URL
+            resolver 9.9.9.9 149.112.112.112 ipv6=off;
+
+            # Unset Authorization and Cookie for security reasons.
+            proxy_set_header Authorization '';
+            proxy_set_header Cookie '';
+
+            # Stops the local disk from being written to (just forwards data through)
+            proxy_max_temp_file_size 0;
+
+            # Proxy the remote file through to the client
+            proxy_pass $takahe_realuri;
+            proxy_ssl_server_name on;
+            add_header X-Takahe-Accel "HIT";
+
+            # Cache these responses too
+            proxy_cache_key $takahe_realuri;
+            proxy_cache_valid 200 304 720h;
+            proxy_cache_valid 301 307 12h;
+            proxy_cache_valid 500 502 503 504 0s;
+            proxy_cache_valid any 72h;
+            add_header X-Cache $upstream_cache_status;
+        }
+
+        # Default config for all other pages
         location / {
-            proxy_redirect   off;
+            proxy_redirect off;
-            proxy_buffering  off;
+            proxy_buffering off;
-            proxy_pass       http://takahe;
+            proxy_pass http://takahe;
         }
     }
 }

mediaproxy/views.py

@@ -1,3 +1,5 @@
+from urllib.parse import urlparse
+
 import httpx
 from django.conf import settings
 from django.http import Http404, HttpResponse
@@ -16,26 +18,42 @@ class BaseProxyView(View):
     def get(self, request, **kwargs):
         self.kwargs = kwargs
         remote_url = self.get_remote_url()
-        try:
+        # See if we can do the nginx trick or a normal forward
-            remote_response = httpx.get(
+        if request.headers.get("x-takahe-accel"):
-                remote_url,
+            bits = urlparse(remote_url)
-                headers={"User-Agent": settings.TAKAHE_USER_AGENT},
+            redirect_url = (
-                follow_redirects=True,
+                f"/__takahe_accel__/{bits.scheme}/{bits.hostname}/{bits.path}"
-                timeout=settings.SETUP.REMOTE_TIMEOUT,
+            )
+            if bits.query:
+                redirect_url += f"?{bits.query}"
+            return HttpResponse(
+                "",
+                headers={
+                    "X-Accel-Redirect": "/__takahe_accel__/",
+                    "X-Takahe-RealUri": remote_url,
+                },
+            )
+        else:
+            try:
+                remote_response = httpx.get(
+                    remote_url,
+                    headers={"User-Agent": settings.TAKAHE_USER_AGENT},
+                    follow_redirects=True,
+                    timeout=settings.SETUP.REMOTE_TIMEOUT,
+                )
+            except httpx.RequestError:
+                return HttpResponse(status=502)
+            if remote_response.status_code >= 400:
+                return HttpResponse(status=502)
+            return HttpResponse(
+                remote_response.content,
+                headers={
+                    "Content-Type": remote_response.headers.get(
+                        "Content-Type", "application/octet-stream"
+                    ),
+                    "Cache-Control": "public, max-age=3600",
+                },
             )
-        except httpx.RequestError:
-            return HttpResponse(status=502)
-        if remote_response.status_code >= 400:
-            return HttpResponse(status=502)
-        return HttpResponse(
-            remote_response.content,
-            headers={
-                "Content-Type": remote_response.headers.get(
-                    "Content-Type", "application/octet-stream"
-                ),
-                "Cache-Control": "public, max-age=3600",
-            },
-        )
 
     def get_remote_url(self):
         raise NotImplementedError()