diff --git a/crawlers/moonstreamcrawlers/cli.py b/crawlers/moonstreamcrawlers/cli.py index 55b55458..e0fc0cf9 100644 --- a/crawlers/moonstreamcrawlers/cli.py +++ b/crawlers/moonstreamcrawlers/cli.py @@ -104,15 +104,18 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None: time.sleep(5) if (len(missing_blocks_numbers_total)) > 0: - if args.lazy: + if bool(strtobool(args.lazy)): + print("Executed lazy block crawler") crawl_blocks( missing_blocks_numbers_total, with_transactions=bool(strtobool(args.transactions)), + verbose=True, ) else: crawl_blocks_executor( missing_blocks_numbers_total, with_transactions=bool(strtobool(args.transactions)), + verbose=True, ) print( f"Required {time.time() - startTime} with {MOONSTREAM_CRAWL_WORKERS} workers " diff --git a/crawlers/moonstreamcrawlers/ethereum.py b/crawlers/moonstreamcrawlers/ethereum.py index bf28a9f4..20b6c921 100644 --- a/crawlers/moonstreamcrawlers/ethereum.py +++ b/crawlers/moonstreamcrawlers/ethereum.py @@ -75,7 +75,9 @@ def get_latest_blocks(with_transactions: bool = False) -> None: return block_latest_exist.block_number, block_latest.number -def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) -> None: +def crawl_blocks( + blocks_numbers: List[int], with_transactions: bool = False, verbose: bool = False +) -> None: """ Open database and geth sessions and fetch block data from blockchain. """ @@ -92,6 +94,9 @@ def crawl_blocks(blocks_numbers: List[int], with_transactions: bool = False) -> db_session.commit() + if verbose: + print(f"Added {block_number} block") + def check_missing_blocks(blocks_numbers: List[int]) -> List[int]: """ @@ -112,15 +117,22 @@ def check_missing_blocks(blocks_numbers: List[int]) -> List[int]: def crawl_blocks_executor( - block_numbers_list: List[int], with_transactions: bool = False + block_numbers_list: List[int], + with_transactions: bool = False, + verbose: bool = False, ) -> None: """ Execute crawler in processes. """ with ProcessPoolExecutor(max_workers=MOONSTREAM_CRAWL_WORKERS) as executor: for worker in range(1, MOONSTREAM_CRAWL_WORKERS + 1): + worker_block_numbers_list = block_numbers_list[ + worker - 1 :: MOONSTREAM_CRAWL_WORKERS + ] + if verbose: + print(f"Spawned process for {len(worker_block_numbers_list)} blocks") executor.submit( crawl_blocks, - block_numbers_list[worker - 1 :: MOONSTREAM_CRAWL_WORKERS], + worker_block_numbers_list, with_transactions, )