From ade954c2cd157d3250a4c602a79c0174b3cf2442 Mon Sep 17 00:00:00 2001 From: Corry Haines Date: Sat, 14 Jan 2023 10:15:18 -0800 Subject: [PATCH] Instruct robots to ignore proxy (#413) --- static/robots.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/static/robots.txt b/static/robots.txt index d910569..ea045a6 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -3,3 +3,6 @@ User-Agent: * # Don't allow any bot to crawl tags. Disallow: /tags/ Disallow: /tags/* + +# Don't allow bots to crawl through the proxy +Disallow: /proxy/*