Improved speed for missing blocks command check

crawlers/moonstreamcrawlers/cli.py

@@ -134,8 +134,10 @@ def ethcrawler_blocks_missing_handler(args: argparse.Namespace) -> None:
         missing_blocks_numbers = check_missing_blocks(
             blocks_numbers=blocks_numbers_list,
         )
+        if len(missing_blocks_numbers) > 0:
+            print(f"Found {len(missing_blocks_numbers)} missing blocks")
         missing_blocks_numbers_total.extend(missing_blocks_numbers)
-    print(f"Found {len(missing_blocks_numbers_total)} missing blocks")
+    print(f"Found {len(missing_blocks_numbers_total)} missing blocks total")
 
     time.sleep(5)
 

crawlers/moonstreamcrawlers/ethereum.py

@@ -119,16 +119,19 @@ def check_missing_blocks(blocks_numbers: List[int]) -> List[int]:
     Query block from postgres. If block does not presented in database,
     add to missing blocks numbers list.
     """
-    missing_blocks_numbers = []
+    bottom_block = min(blocks_numbers[-1], blocks_numbers[0])
-    for block_number in blocks_numbers:
+    top_block = max(blocks_numbers[-1], blocks_numbers[0])
-        with yield_db_session_ctx() as db_session:
+    with yield_db_session_ctx() as db_session:
-            block_exist = (
+        blocks_exist_raw = (
-                db_session.query(EthereumBlock.block_number)
+            db_session.query(EthereumBlock.block_number)
-                .filter(EthereumBlock.block_number == block_number)
+            .filter(EthereumBlock.block_number >= bottom_block)
-                .one_or_none()
+            .filter(EthereumBlock.block_number <= top_block)
-            )
+            .all()
-            if block_exist is None:
+        )
-                missing_blocks_numbers.append(block_number)
+    blocks_exist = [block[0] for block in blocks_exist_raw]
+    missing_blocks_numbers = [
+        block for block in blocks_numbers if block not in blocks_exist
+    ]
     return missing_blocks_numbers
 
 
@@ -177,6 +180,7 @@ def crawl_blocks_executor(
             for error in errors:
                 print(f"- {error}")
 
+
 def process_contract_deployments() -> List[Tuple[str, str]]:
     """
     Checks for new smart contracts that have been deployed to the blockchain but not registered in