kopia lustrzana https://github.com/cblgh/lieu
go fmt
rodzic
9517f62de2
commit
9377bd6fab
|
@ -144,7 +144,7 @@ func Ingest(config types.Config) {
|
|||
page.AboutSource = token
|
||||
processed = partitionSentence(payload)
|
||||
case "para":
|
||||
if (page.AboutSource != "og-desc" || len(rawdata)*10 > len(page.About)*7) {
|
||||
if page.AboutSource != "og-desc" || len(rawdata)*10 > len(page.About)*7 {
|
||||
if performAboutHeuristic(config.Data.Heuristics, payload) {
|
||||
page.About = rawdata
|
||||
page.AboutSource = token
|
||||
|
@ -205,7 +205,9 @@ func ingestBatch(db *sql.DB, batch []types.SearchFragment, pageMap map[string]ty
|
|||
database.InsertManyPages(db, pages)
|
||||
for i := 0; i < len(batch); i += 3000 {
|
||||
end_i := i + 3000
|
||||
if (end_i > len(batch)) { end_i = len(batch)}
|
||||
if end_i > len(batch) {
|
||||
end_i = len(batch)
|
||||
}
|
||||
database.InsertManyWords(db, batch[i:end_i])
|
||||
}
|
||||
database.InsertManyExternalLinks(db, links)
|
||||
|
|
Ładowanie…
Reference in New Issue