kopia lustrzana https://github.com/Stopka/fedicrawl
Changed tag from noindex to nobot
rodzic
c0b10f2e7a
commit
59f49ce29d
|
@ -13,7 +13,7 @@ export const refreshFeedsOnPage = async (
|
||||||
robotsTxt: RobotsTxt
|
robotsTxt: RobotsTxt
|
||||||
): Promise<Feed[]> => {
|
): Promise<Feed[]> => {
|
||||||
const feedData = await provider.retrieveFeeds(node.domain, page, robotsTxt)
|
const feedData = await provider.retrieveFeeds(node.domain, page, robotsTxt)
|
||||||
const indexableFeedData = feedData.filter(item => item.indexable && !item.description.includes('#noindex'))
|
const indexableFeedData = feedData.filter(item => item.indexable && !item.description.includes('#nobot'))
|
||||||
console.info('Retrieved feeds', {
|
console.info('Retrieved feeds', {
|
||||||
count: feedData.length,
|
count: feedData.length,
|
||||||
indexableCount: indexableFeedData.length,
|
indexableCount: indexableFeedData.length,
|
||||||
|
|
Ładowanie…
Reference in New Issue