diff --git a/public-staging/robots.txt b/public-staging/robots.txt new file mode 100644 index 00000000..1f53798b --- /dev/null +++ b/public-staging/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/public/robots.txt b/public/robots.txt index c2a49f4f..ffb4c1c6 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,2 +1,23 @@ User-agent: * Allow: / + +# Disallow authenticated pages + +Disallow /intent +Disallow /settings +Disallow /blocks +Disallow /bookmarks +Disallow /compose +Disallow /conversations +Disallow /domain_blocks +Disallow /favourites +Disallow /home +Disallow /mutes +Disallow /notifications +Disallow /pinned +Disallow /search +Disallow /settings +Disallow /share-target + +# Wait 1 second between successive requests. +Crawl-delay: 1