fix: disallow all indexing on canary site and indexing of auth-only pages in prod (#1175)

public-staging/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

public/robots.txt

@@ -1,2 +1,23 @@
 User-agent: *
 Allow: /
+
+# Disallow authenticated pages
+
+Disallow /intent
+Disallow /settings
+Disallow /blocks
+Disallow /bookmarks
+Disallow /compose
+Disallow /conversations
+Disallow /domain_blocks
+Disallow /favourites
+Disallow /home
+Disallow /mutes
+Disallow /notifications
+Disallow /pinned
+Disallow /search
+Disallow /settings
+Disallow /share-target
+
+# Wait 1 second between successive requests.
+Crawl-delay: 1