It respects noindex option in mastodon

application/src/Fediverse/Providers/FeedData.ts

@@ -19,4 +19,5 @@ export interface FeedData {
     name: string
     hostDomain: string
   }
+  indexable: boolean
 }

application/src/Fediverse/Providers/Mastodon/retrieveLocalPublicUsersPage.ts

@@ -34,7 +34,8 @@ const schema = z.array(
         value: z.string(),
         verified_at: z.nullable(z.string())
       })
-    )
+    ),
+    noindex: z.boolean().optional().nullable()
   })
 )
 
@@ -94,7 +95,8 @@ export const retrieveLocalPublicUsersPage: FeedProviderMethod = async (
         }
       }),
       type: 'account',
-      parentFeed: undefined
+      parentFeed: undefined,
+      indexable: !(item.noindex ?? false)
     }
   })
 }

application/src/Fediverse/Providers/Misskey/retrieveUsersPage.ts

@@ -121,7 +121,8 @@ export const retrieveUsersPage: FeedProviderMethod = async (
         ].filter((field) => field.value !== null) as FieldData[])
       ],
       type: 'account',
-      parentFeed: undefined
+      parentFeed: undefined,
+      indexable: true
     }
   })
 }

application/src/Fediverse/Providers/Peertube/retrieveAccounts.ts

@@ -60,7 +60,8 @@ export const retrieveAccounts: FeedProviderMethod = async (domain, page, robotsT
         lastStatusAt: undefined,
         statusesCount: undefined,
         type: 'account',
-        parentFeed: undefined
+        parentFeed: undefined,
+        indexable: true
       }
     })
 }

application/src/Fediverse/Providers/Peertube/retrieveVideoChannels.ts

@@ -78,7 +78,8 @@ export const retrieveVideoChannels: FeedProviderMethod = async (
         parentFeed: {
           name: item.ownerAccount.name,
           hostDomain: item.ownerAccount.host
-        }
+        },
+        indexable: true
       }
     })
 }

application/src/Jobs/Feeds/refreshFeedsOnPage.ts

@@ -13,14 +13,16 @@ export const refreshFeedsOnPage = async (
   robotsTxt: RobotsTxt
 ): Promise<Feed[]> => {
   const feedData = await provider.retrieveFeeds(node.domain, page, robotsTxt)
+  const indexableFeedData = feedData.filter(item => item.indexable)
   console.info('Retrieved feeds', {
     count: feedData.length,
+    indexableCount: indexableFeedData.length,
     domain: node.domain,
     provider: provider.getKey(),
     page
   })
   return await Promise.all(
-    feedData.map(
+    indexableFeedData.map(
       async (feedDataItem) =>
         await refreshOrAddFeed(elastic, node, feedDataItem)
     )