kopia lustrzana https://github.com/Stopka/fedicrawl
It respects noindex option in mastodon
rodzic
704c7c066e
commit
4113d78a17
|
@ -19,4 +19,5 @@ export interface FeedData {
|
||||||
name: string
|
name: string
|
||||||
hostDomain: string
|
hostDomain: string
|
||||||
}
|
}
|
||||||
|
indexable: boolean
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,7 +34,8 @@ const schema = z.array(
|
||||||
value: z.string(),
|
value: z.string(),
|
||||||
verified_at: z.nullable(z.string())
|
verified_at: z.nullable(z.string())
|
||||||
})
|
})
|
||||||
)
|
),
|
||||||
|
noindex: z.boolean().optional().nullable()
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -94,7 +95,8 @@ export const retrieveLocalPublicUsersPage: FeedProviderMethod = async (
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
type: 'account',
|
type: 'account',
|
||||||
parentFeed: undefined
|
parentFeed: undefined,
|
||||||
|
indexable: !(item.noindex ?? false)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -121,7 +121,8 @@ export const retrieveUsersPage: FeedProviderMethod = async (
|
||||||
].filter((field) => field.value !== null) as FieldData[])
|
].filter((field) => field.value !== null) as FieldData[])
|
||||||
],
|
],
|
||||||
type: 'account',
|
type: 'account',
|
||||||
parentFeed: undefined
|
parentFeed: undefined,
|
||||||
|
indexable: true
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -60,7 +60,8 @@ export const retrieveAccounts: FeedProviderMethod = async (domain, page, robotsT
|
||||||
lastStatusAt: undefined,
|
lastStatusAt: undefined,
|
||||||
statusesCount: undefined,
|
statusesCount: undefined,
|
||||||
type: 'account',
|
type: 'account',
|
||||||
parentFeed: undefined
|
parentFeed: undefined,
|
||||||
|
indexable: true
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -78,7 +78,8 @@ export const retrieveVideoChannels: FeedProviderMethod = async (
|
||||||
parentFeed: {
|
parentFeed: {
|
||||||
name: item.ownerAccount.name,
|
name: item.ownerAccount.name,
|
||||||
hostDomain: item.ownerAccount.host
|
hostDomain: item.ownerAccount.host
|
||||||
}
|
},
|
||||||
|
indexable: true
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,14 +13,16 @@ export const refreshFeedsOnPage = async (
|
||||||
robotsTxt: RobotsTxt
|
robotsTxt: RobotsTxt
|
||||||
): Promise<Feed[]> => {
|
): Promise<Feed[]> => {
|
||||||
const feedData = await provider.retrieveFeeds(node.domain, page, robotsTxt)
|
const feedData = await provider.retrieveFeeds(node.domain, page, robotsTxt)
|
||||||
|
const indexableFeedData = feedData.filter(item => item.indexable)
|
||||||
console.info('Retrieved feeds', {
|
console.info('Retrieved feeds', {
|
||||||
count: feedData.length,
|
count: feedData.length,
|
||||||
|
indexableCount: indexableFeedData.length,
|
||||||
domain: node.domain,
|
domain: node.domain,
|
||||||
provider: provider.getKey(),
|
provider: provider.getKey(),
|
||||||
page
|
page
|
||||||
})
|
})
|
||||||
return await Promise.all(
|
return await Promise.all(
|
||||||
feedData.map(
|
indexableFeedData.map(
|
||||||
async (feedDataItem) =>
|
async (feedDataItem) =>
|
||||||
await refreshOrAddFeed(elastic, node, feedDataItem)
|
await refreshOrAddFeed(elastic, node, feedDataItem)
|
||||||
)
|
)
|
||||||
|
|
Ładowanie…
Reference in New Issue