From 4113d78a17eb1e11a0884018be525a5f9401f67e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20=C5=A0korpil?= <stepan@skorpil.cz>
Date: Tue, 22 Nov 2022 17:01:25 +0100
Subject: [PATCH] It respects noindex option in mastodon

---
 application/src/Fediverse/Providers/FeedData.ts             | 1 +
 .../Providers/Mastodon/retrieveLocalPublicUsersPage.ts      | 6 ++++--
 .../src/Fediverse/Providers/Misskey/retrieveUsersPage.ts    | 3 ++-
 .../src/Fediverse/Providers/Peertube/retrieveAccounts.ts    | 3 ++-
 .../Fediverse/Providers/Peertube/retrieveVideoChannels.ts   | 3 ++-
 application/src/Jobs/Feeds/refreshFeedsOnPage.ts            | 4 +++-
 6 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/application/src/Fediverse/Providers/FeedData.ts b/application/src/Fediverse/Providers/FeedData.ts
index 3ca3b38..59dd8e0 100644
--- a/application/src/Fediverse/Providers/FeedData.ts
+++ b/application/src/Fediverse/Providers/FeedData.ts
@@ -19,4 +19,5 @@ export interface FeedData {
     name: string
     hostDomain: string
   }
+  indexable: boolean
 }
diff --git a/application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts b/application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts
index facdf9d..2df82ca 100644
--- a/application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts
+++ b/application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts
@@ -34,7 +34,8 @@ const schema = z.array(
         value: z.string(),
         verified_at: z.nullable(z.string())
       })
-    )
+    ),
+    noindex: z.boolean().optional().nullable()
   })
 )
 
@@ -94,7 +95,8 @@ export const retrieveLocalPublicUsersPage: FeedProviderMethod = async (
         }
       }),
       type: 'account',
-      parentFeed: undefined
+      parentFeed: undefined,
+      indexable: !(item.noindex ?? false)
     }
   })
 }
diff --git a/application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts b/application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts
index 6693ee9..e360709 100644
--- a/application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts
+++ b/application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts
@@ -121,7 +121,8 @@ export const retrieveUsersPage: FeedProviderMethod = async (
         ].filter((field) => field.value !== null) as FieldData[])
       ],
       type: 'account',
-      parentFeed: undefined
+      parentFeed: undefined,
+      indexable: true
     }
   })
 }
diff --git a/application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts b/application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts
index e193d12..4007dfd 100644
--- a/application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts
+++ b/application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts
@@ -60,7 +60,8 @@ export const retrieveAccounts: FeedProviderMethod = async (domain, page, robotsT
         lastStatusAt: undefined,
         statusesCount: undefined,
         type: 'account',
-        parentFeed: undefined
+        parentFeed: undefined,
+        indexable: true
       }
     })
 }
diff --git a/application/src/Fediverse/Providers/Peertube/retrieveVideoChannels.ts b/application/src/Fediverse/Providers/Peertube/retrieveVideoChannels.ts
index c975447..21a007e 100644
--- a/application/src/Fediverse/Providers/Peertube/retrieveVideoChannels.ts
+++ b/application/src/Fediverse/Providers/Peertube/retrieveVideoChannels.ts
@@ -78,7 +78,8 @@ export const retrieveVideoChannels: FeedProviderMethod = async (
         parentFeed: {
           name: item.ownerAccount.name,
           hostDomain: item.ownerAccount.host
-        }
+        },
+        indexable: true
       }
     })
 }
diff --git a/application/src/Jobs/Feeds/refreshFeedsOnPage.ts b/application/src/Jobs/Feeds/refreshFeedsOnPage.ts
index 79c9e31..fe400ed 100644
--- a/application/src/Jobs/Feeds/refreshFeedsOnPage.ts
+++ b/application/src/Jobs/Feeds/refreshFeedsOnPage.ts
@@ -13,14 +13,16 @@ export const refreshFeedsOnPage = async (
   robotsTxt: RobotsTxt
 ): Promise<Feed[]> => {
   const feedData = await provider.retrieveFeeds(node.domain, page, robotsTxt)
+  const indexableFeedData = feedData.filter(item => item.indexable)
   console.info('Retrieved feeds', {
     count: feedData.length,
+    indexableCount: indexableFeedData.length,
     domain: node.domain,
     provider: provider.getKey(),
     page
   })
   return await Promise.all(
-    feedData.map(
+    indexableFeedData.map(
       async (feedDataItem) =>
         await refreshOrAddFeed(elastic, node, feedDataItem)
     )