From 5ec29305d78db4954daba43ff8dfe15152282166 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Thu, 29 Jul 2021 11:36:19 -0700 Subject: [PATCH] Cleaned up a little the worker submissions for ethcrawler --- crawlers/moonstreamcrawlers/ethereum.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crawlers/moonstreamcrawlers/ethereum.py b/crawlers/moonstreamcrawlers/ethereum.py index ade7c26a..a2e51999 100644 --- a/crawlers/moonstreamcrawlers/ethereum.py +++ b/crawlers/moonstreamcrawlers/ethereum.py @@ -124,15 +124,17 @@ def crawl_blocks_executor( """ Execute crawler in processes. """ + worker_indices = range(MOONSTREAM_CRAWL_WORKERS) + worker_job_lists = [[] for _ in worker_indices] + for i, block_number in enumerate(block_numbers_list): + worker_job_lists[i % MOONSTREAM_CRAWL_WORKERS].append(block_number) + with ProcessPoolExecutor(max_workers=MOONSTREAM_CRAWL_WORKERS) as executor: - for worker in range(1, MOONSTREAM_CRAWL_WORKERS + 1): - worker_block_numbers_list = block_numbers_list[ - worker - 1 :: MOONSTREAM_CRAWL_WORKERS - ] + for worker in worker_indices: if verbose: - print(f"Spawned process for {len(worker_block_numbers_list)} blocks") + print(f"Spawned process for {len(worker_job_lists[worker])} blocks") executor.submit( crawl_blocks, - worker_block_numbers_list, + worker_job_lists[worker], with_transactions, )