From 45455a6ac25853451d380fa18f8b1a975b10d3d1 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 3 Aug 2021 11:41:51 +0300 Subject: [PATCH] Improved speed for missing blocks command check --- crawlers/moonstreamcrawlers/cli.py | 4 +++- crawlers/moonstreamcrawlers/ethereum.py | 24 ++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/crawlers/moonstreamcrawlers/cli.py b/crawlers/moonstreamcrawlers/cli.py index 2b72051a..7d08e1ec 100644 --- a/crawlers/moonstreamcrawlers/cli.py +++ b/crawlers/moonstreamcrawlers/cli.py @@ -134,8 +134,10 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None: missing_blocks_numbers = check_missing_blocks( blocks_numbers=blocks_numbers_list, ) + if len(missing_blocks_numbers) > 0: + print(f"Found {len(missing_blocks_numbers)} missing blocks") missing_blocks_numbers_total.extend(missing_blocks_numbers) - print(f"Found {len(missing_blocks_numbers_total)} missing blocks") + print(f"Found {len(missing_blocks_numbers_total)} missing blocks total") time.sleep(5) diff --git a/crawlers/moonstreamcrawlers/ethereum.py b/crawlers/moonstreamcrawlers/ethereum.py index 58c4b30f..f385fda5 100644 --- a/crawlers/moonstreamcrawlers/ethereum.py +++ b/crawlers/moonstreamcrawlers/ethereum.py @@ -119,16 +119,19 @@ def check_missing_blocks(blocks_numbers: List[int]) -> List[int]: Query block from postgres. If block does not presented in database, add to missing blocks numbers list. """ - missing_blocks_numbers = [] - for block_number in blocks_numbers: - with yield_db_session_ctx() as db_session: - block_exist = ( - db_session.query(EthereumBlock.block_number) - .filter(EthereumBlock.block_number == block_number) - .one_or_none() - ) - if block_exist is None: - missing_blocks_numbers.append(block_number) + bottom_block = min(blocks_numbers[-1], blocks_numbers[0]) + top_block = max(blocks_numbers[-1], blocks_numbers[0]) + with yield_db_session_ctx() as db_session: + blocks_exist_raw = ( + db_session.query(EthereumBlock.block_number) + .filter(EthereumBlock.block_number >= bottom_block) + .filter(EthereumBlock.block_number <= top_block) + .all() + ) + blocks_exist = [block[0] for block in blocks_exist_raw] + missing_blocks_numbers = [ + block for block in blocks_numbers if block not in blocks_exist + ] return missing_blocks_numbers @@ -177,6 +180,7 @@ def crawl_blocks_executor( for error in errors: print(f"- {error}") + def process_contract_deployments() -> List[Tuple[str, str]]: """ Checks for new smart contracts that have been deployed to the blockchain but not registered in