--confirmations argument to "ethcrawl synchronize"

crawlers/mooncrawl/cli.py

@@ -82,9 +82,7 @@ def ethcrawler_blocks_sync_handler(args: argparse.Namespace) -> None:
     """
     starting_block: int = args.start
     while True:
-        bottom_block_number, top_block_number = get_latest_blocks(
-            with_transactions=not args.notransactions
-        )
+        bottom_block_number, top_block_number = get_latest_blocks(args.confirmations)
         bottom_block_number = max(bottom_block_number + 1, starting_block)
         if bottom_block_number >= top_block_number:
             print(
@@ -218,6 +216,13 @@ def main() -> None:
         default=0,
         help="(Optional) Block to start synchronization from. Default: 0",
     )
+    parser_ethcrawler_blocks_sync.add_argument(
+        "-c",
+        "--confirmations",
+        type=int,
+        default=0,
+        help="Number of confirmations we require before storing a block in the database. (Default: 0)",
+    )
     parser_ethcrawler_blocks_sync.add_argument(
         "--order",
         type=processing_order,

crawlers/mooncrawl/ethereum.py

@@ -71,24 +71,29 @@ def add_block_transactions(db_session, block: BlockData) -> None:
         db_session.add(tx_obj)
 
 
-def get_latest_blocks(with_transactions: bool = False) -> Tuple[Optional[int], int]:
+def get_latest_blocks(confirmations: int = 0) -> Tuple[Optional[int], int]:
+    """
+    Retrieve the latest block from the connected node (connection is created by the connect() method).
+
+    If confirmations > 0, and the latest block on the node has block number N, this returns the block
+    with block_number (N - confirmations)
+    """
     web3_client = connect()
-    block_latest: BlockData = web3_client.eth.get_block(
-        "latest", full_transactions=with_transactions
-    )
+    latest_block_number: int = web3_client.eth.block_number
+    if confirmations > 0:
+        latest_block_number -= confirmations
+
     with yield_db_session_ctx() as db_session:
-        block_number_latest_exist_row = (
+        latest_stored_block_row = (
             db_session.query(EthereumBlock.block_number)
             .order_by(EthereumBlock.block_number.desc())
             .first()
         )
-        block_number_latest_exist = (
-            None
-            if block_number_latest_exist_row is None
-            else block_number_latest_exist_row[0]
+        latest_stored_block_number = (
+            None if latest_stored_block_row is None else latest_stored_block_row[0]
         )
 
-    return block_number_latest_exist, block_latest.number
+    return latest_stored_block_number, latest_block_number
 
 
 def crawl_blocks(
@@ -143,6 +148,12 @@ def crawl_blocks_executor(
 ) -> None:
     """
     Execute crawler in processes.
+
+    Args:
+    block_numbers_list - List of block numbers to add to database.
+    with_transactions - If True, also adds transactions from those blocks to the ethereum_transactions table.
+    verbose - Print logs to stdout?
+    num_processes - Number of processes to use to feed blocks into database.
     """
     errors: List[Exception] = []