kopia lustrzana https://github.com/Stopka/fedicrawl
15 wiersze
760 B
TypeScript
15 wiersze
760 B
TypeScript
import { createMissingNodes } from '../../Storage/Nodes/createMissingNodes'
|
|
import { NodeProvider } from '../../Fediverse/Providers/NodeProvider'
|
|
import Node from '../../Storage/Definitions/Node'
|
|
import { ElasticClient } from '../../Storage/ElasticClient'
|
|
import isDomainNotBanned from '../../Storage/Nodes/isDomainNotBanned'
|
|
|
|
export const findNewNodesOnPage = async (
|
|
elastic: ElasticClient, provider: NodeProvider, node:Node, page:number
|
|
):Promise<number> => {
|
|
let domains = await provider.retrieveNodes(node.domain, page)
|
|
domains = domains.filter(isDomainNotBanned)
|
|
console.log('Found nodes', { count: domains.length, domain: node.domain, provider: provider.getKey(), page: page })
|
|
return await createMissingNodes(elastic, domains, node.domain)
|
|
}
|