From 5a6248a08faa43aa82474adf2648b749ac8949f0 Mon Sep 17 00:00:00 2001
From: Daniel Roe <daniel@roe.dev>
Date: Sun, 15 Jan 2023 14:34:23 +0000
Subject: [PATCH] fix: disallow all indexing on canary site and indexing of
 auth-only pages in prod (#1175)

---
 public-staging/robots.txt |  2 ++
 public/robots.txt         | 21 +++++++++++++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 public-staging/robots.txt

diff --git a/public-staging/robots.txt b/public-staging/robots.txt
new file mode 100644
index 00000000..1f53798b
--- /dev/null
+++ b/public-staging/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/public/robots.txt b/public/robots.txt
index c2a49f4f..ffb4c1c6 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,2 +1,23 @@
 User-agent: *
 Allow: /
+
+# Disallow authenticated pages
+
+Disallow /intent
+Disallow /settings
+Disallow /blocks
+Disallow /bookmarks
+Disallow /compose
+Disallow /conversations
+Disallow /domain_blocks
+Disallow /favourites
+Disallow /home
+Disallow /mutes
+Disallow /notifications
+Disallow /pinned
+Disallow /search
+Disallow /settings
+Disallow /share-target
+
+# Wait 1 second between successive requests.
+Crawl-delay: 1