Error handling for crawlers

pull/35/head
Neeraj Kashyap 2021-07-29 11:52:37 -07:00
rodzic cbd9c2ccf1
commit d809d9b3d5
1 zmienionych plików z 11 dodań i 0 usunięć

Wyświetl plik

@ -124,6 +124,13 @@ def crawl_blocks_executor(
"""
Execute crawler in processes.
"""
errors: List[Exception] = []
def record_error(f: Future) -> None:
error = f.exception()
if error is not None:
errors.append(error)
worker_indices = range(MOONSTREAM_CRAWL_WORKERS)
worker_job_lists = [[] for _ in worker_indices]
for i, block_number in enumerate(block_numbers_list):
@ -140,6 +147,10 @@ def crawl_blocks_executor(
worker_job_lists[worker],
with_transactions,
)
result.add_done_callback(record_error)
results.append(result)
wait(results)
print("Errors:")
for error in errors:
print(f"- {error}")