kopia lustrzana https://github.com/Stopka/fedicrawl
Add display name to fulltext field, fixed waiting when all feeds are processed
rodzic
3acd0ec72f
commit
d2ab8bfb2f
|
@ -3,7 +3,8 @@ import striptags from 'striptags'
|
||||||
|
|
||||||
export default function (feedData: FeedData):string {
|
export default function (feedData: FeedData):string {
|
||||||
return striptags(
|
return striptags(
|
||||||
feedData.description +
|
feedData.displayName +
|
||||||
|
' ' + feedData.description +
|
||||||
' ' + feedData.fields.map(field => field.name).join(' ') +
|
' ' + feedData.fields.map(field => field.name).join(' ') +
|
||||||
' ' + feedData.fields.map(field => field.value).join(' ')
|
' ' + feedData.fields.map(field => field.value).join(' ')
|
||||||
)
|
)
|
||||||
|
|
|
@ -3,17 +3,22 @@ import prismaClient from './Storage/PrismaClient'
|
||||||
import { addNodeSeed } from './Jobs/Seed/addNodeSeed'
|
import { addNodeSeed } from './Jobs/Seed/addNodeSeed'
|
||||||
import { processNextNode } from './Jobs/processNextNode'
|
import { processNextNode } from './Jobs/processNextNode'
|
||||||
|
|
||||||
const loop = async ():Promise<void> => {
|
const loop = async (): Promise<void> => {
|
||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
await processNextNode(prismaClient, providerRegistry)
|
await processNextNode(prismaClient, providerRegistry)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setTimeout(loop, 1000 * 60 * 60)
|
console.warn(err)
|
||||||
|
const milisecondsInMinute = 1000 * 60
|
||||||
|
const timeout = 60 * milisecondsInMinute
|
||||||
|
console.info('Delaying next node process', { timeoutMinutes: timeout / milisecondsInMinute, now: new Date() })
|
||||||
|
setTimeout(loop, timeout)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const app = async ():Promise<void> => {
|
const app = async (): Promise<void> => {
|
||||||
const seedDomain = process.env.SEED_NODE_DOMAIN ?? 'mastodon.social'
|
const seedDomain = process.env.SEED_NODE_DOMAIN ?? 'mastodon.social'
|
||||||
await addNodeSeed(prismaClient, seedDomain)
|
await addNodeSeed(prismaClient, seedDomain)
|
||||||
setTimeout(loop)
|
setTimeout(loop)
|
||||||
|
|
Ładowanie…
Reference in New Issue