pull/15/head
cblgh 2022-11-22 14:08:59 +01:00
rodzic 9517f62de2
commit 9377bd6fab
3 zmienionych plików z 19 dodań i 17 usunięć

Wyświetl plik

@ -144,7 +144,7 @@ func Ingest(config types.Config) {
page.AboutSource = token
processed = partitionSentence(payload)
case "para":
if (page.AboutSource != "og-desc" || len(rawdata)*10 > len(page.About)*7) {
if page.AboutSource != "og-desc" || len(rawdata)*10 > len(page.About)*7 {
if performAboutHeuristic(config.Data.Heuristics, payload) {
page.About = rawdata
page.AboutSource = token
@ -205,7 +205,9 @@ func ingestBatch(db *sql.DB, batch []types.SearchFragment, pageMap map[string]ty
database.InsertManyPages(db, pages)
for i := 0; i < len(batch); i += 3000 {
end_i := i + 3000
if (end_i > len(batch)) { end_i = len(batch)}
if end_i > len(batch) {
end_i = len(batch)
}
database.InsertManyWords(db, batch[i:end_i])
}
database.InsertManyExternalLinks(db, links)