From c7409b3500d59dd85a60af79e542268a06bca815 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Mon, 19 Dec 2022 07:26:38 +0000 Subject: [PATCH] Get nginx to proxy stuff for us! --- .gitignore | 2 +- docker/nginx.conf | 63 +++++++++++++++++++++++++++++++++++++-------- mediaproxy/views.py | 56 ++++++++++++++++++++++++++-------------- 3 files changed, 90 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index fd7ddf6..fec58af 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ .idea/* .venv .vscode -/*.env +/*.env* /build /cache/ /docs/_build diff --git a/docker/nginx.conf b/docker/nginx.conf index a4ce277..9187d87 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -15,6 +15,8 @@ http { server "127.0.0.1:8001"; } + # access_log /dev/stdout; + server { listen 8000; listen [::]:8000; @@ -26,10 +28,7 @@ http { ignore_invalid_headers on; proxy_connect_timeout 900; - proxy_headers_hash_max_size 1024; - proxy_headers_hash_bucket_size 128; - - client_max_body_size 512M; + client_max_body_size 100M; client_body_buffer_size 128k; charset utf-8; @@ -38,26 +37,68 @@ http { proxy_http_version 1.1; proxy_cache takahe; + # Serves static files from the collected dir location /static/ { alias /takahe/static-collected/; } + # Proxies media and remote media with caching location ~* ^/(media|proxy) { + # Cache media and proxied resources proxy_cache_key $host$uri; - proxy_cache_valid 200 304 720h; - proxy_cache_valid 301 307 12h; + proxy_cache_valid 200 304 4h; + proxy_cache_valid 301 307 4h; proxy_cache_valid 500 502 503 504 0s; - proxy_cache_valid any 72h; - + proxy_cache_valid any 1h; add_header X-Cache $upstream_cache_status; + # Signal to Takahē that we support full URI accel proxying + proxy_set_header X-Takahe-Accel true; + proxy_pass http://takahe; } + # Internal target for X-Accel redirects that stashes the URI in a var + location /__takahe_accel__/ { + internal; + set $takahe_realuri $upstream_http_x_takahe_realuri; + rewrite ^/(.+) /__takahe_accel__/real/; + } + + # Real internal-only target for X-Accel redirects + location /__takahe_accel__/real/ { + # Only allow internal redirects + internal; + + # Reconstruct the remote URL + resolver 9.9.9.9 149.112.112.112 ipv6=off; + + # Unset Authorization and Cookie for security reasons. + proxy_set_header Authorization ''; + proxy_set_header Cookie ''; + + # Stops the local disk from being written to (just forwards data through) + proxy_max_temp_file_size 0; + + # Proxy the remote file through to the client + proxy_pass $takahe_realuri; + proxy_ssl_server_name on; + add_header X-Takahe-Accel "HIT"; + + # Cache these responses too + proxy_cache_key $takahe_realuri; + proxy_cache_valid 200 304 720h; + proxy_cache_valid 301 307 12h; + proxy_cache_valid 500 502 503 504 0s; + proxy_cache_valid any 72h; + add_header X-Cache $upstream_cache_status; + } + + # Default config for all other pages location / { - proxy_redirect off; - proxy_buffering off; - proxy_pass http://takahe; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://takahe; } } } diff --git a/mediaproxy/views.py b/mediaproxy/views.py index c745e7f..12cfd7c 100644 --- a/mediaproxy/views.py +++ b/mediaproxy/views.py @@ -1,3 +1,5 @@ +from urllib.parse import urlparse + import httpx from django.conf import settings from django.http import Http404, HttpResponse @@ -16,26 +18,42 @@ class BaseProxyView(View): def get(self, request, **kwargs): self.kwargs = kwargs remote_url = self.get_remote_url() - try: - remote_response = httpx.get( - remote_url, - headers={"User-Agent": settings.TAKAHE_USER_AGENT}, - follow_redirects=True, - timeout=settings.SETUP.REMOTE_TIMEOUT, + # See if we can do the nginx trick or a normal forward + if request.headers.get("x-takahe-accel"): + bits = urlparse(remote_url) + redirect_url = ( + f"/__takahe_accel__/{bits.scheme}/{bits.hostname}/{bits.path}" + ) + if bits.query: + redirect_url += f"?{bits.query}" + return HttpResponse( + "", + headers={ + "X-Accel-Redirect": "/__takahe_accel__/", + "X-Takahe-RealUri": remote_url, + }, + ) + else: + try: + remote_response = httpx.get( + remote_url, + headers={"User-Agent": settings.TAKAHE_USER_AGENT}, + follow_redirects=True, + timeout=settings.SETUP.REMOTE_TIMEOUT, + ) + except httpx.RequestError: + return HttpResponse(status=502) + if remote_response.status_code >= 400: + return HttpResponse(status=502) + return HttpResponse( + remote_response.content, + headers={ + "Content-Type": remote_response.headers.get( + "Content-Type", "application/octet-stream" + ), + "Cache-Control": "public, max-age=3600", + }, ) - except httpx.RequestError: - return HttpResponse(status=502) - if remote_response.status_code >= 400: - return HttpResponse(status=502) - return HttpResponse( - remote_response.content, - headers={ - "Content-Type": remote_response.headers.get( - "Content-Type", "application/octet-stream" - ), - "Cache-Control": "public, max-age=3600", - }, - ) def get_remote_url(self): raise NotImplementedError()