kopia lustrzana https://github.com/bugout-dev/moonstream
Add missing version.
rodzic
38947e1379
commit
b3aa979d78
|
@ -504,7 +504,7 @@ def get_all_entries_from_search(
|
||||||
limit=limit,
|
limit=limit,
|
||||||
offset=offset,
|
offset=offset,
|
||||||
)
|
)
|
||||||
results.extend(existing_methods.results) # type: ignore
|
results.extend(existing_methods.results) # type: ignore
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
@ -573,6 +573,7 @@ def apply_moonworm_tasks(
|
||||||
f"moonworm_task_pickedup:False", # True if task picked up by moonworm-crawler(default each 120 sec)
|
f"moonworm_task_pickedup:False", # True if task picked up by moonworm-crawler(default each 120 sec)
|
||||||
f"historical_crawl_status:pending", # pending, in_progress, done
|
f"historical_crawl_status:pending", # pending, in_progress, done
|
||||||
f"progress:0", # 0-100 %
|
f"progress:0", # 0-100 %
|
||||||
|
f"version:2.0",
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
|
@ -86,18 +86,22 @@ def fill_missing_selectors_in_moonworm_tasks() -> None:
|
||||||
|
|
||||||
logger.info(f"Found {count} missing selectors in batch {len(task_batch)} tasks")
|
logger.info(f"Found {count} missing selectors in batch {len(task_batch)} tasks")
|
||||||
|
|
||||||
## update entries
|
# ## update entries tags in batch
|
||||||
|
|
||||||
try:
|
for batch in [
|
||||||
bc.create_entries_tags(
|
entries_tags[i : i + batch_size]
|
||||||
journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
|
for i in range(0, len(entries_tags), batch_size)
|
||||||
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
]:
|
||||||
entries_tags=entries_tags,
|
try:
|
||||||
timeout=15,
|
bc.create_entries_tags(
|
||||||
)
|
journal_id=MOONSTREAM_MOONWORM_TASKS_JOURNAL,
|
||||||
except BugoutResponseException as e:
|
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||||
logger.error(f"Unable to update entries tags: {e}")
|
entries_tags=batch,
|
||||||
continue
|
timeout=15,
|
||||||
|
)
|
||||||
|
except BugoutResponseException as e:
|
||||||
|
logger.error(f"Unable to update entries tags: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
|
||||||
def deduplicate_moonworm_task_by_selector():
|
def deduplicate_moonworm_task_by_selector():
|
||||||
|
|
Ładowanie…
Reference in New Issue