Merge pull request #68 from bugout-dev/only-events

added `--only events` arg to crawl events faster

README.md

@@ -14,7 +14,7 @@ Moonworm is a set of tools that helps you develop/analyze blockchain dapps. Pump
 
 ## Setup:
 
-```sql
+```bash
 pip install moonworm 
 ```
 
@@ -33,15 +33,18 @@ Arguments:
 - `--contract/-c CONTRACT` Contract address
 - `--web3/-w WEB3`    Web3 provider uri
 - `--start/-s START`  block to start watching
-- `--end/-e END`      block to stop crawling, if not given, crawler will not stop
 
 Optional args:
-
+- `--end/-e END`      block to stop crawling, if not given, crawler will not stop
 - `--poa` Flag for `PoA` networks, for example `polygon`
 - `--confirmations CONFIRMATIONS`  Number of confirmations to set for watch. (Default 12)
 - `--outfile/-o OUTFILE`  `JSONL` file into which to write events and transactions
 - `--db`  Use Moonstream database specified by `MOONSTREAM_DB_URI` to get blocks/transactions. If set, need also provide `--network`
 - `-network {ethereum,polygon}`Network name that represents models from db. If the `--db` is set, required
+- `--only-events` Flag, if set: only watches events. Default=`False`
+- `--min-blocks-batch MIN_BLOCKS_BATCH` Minimum number of blocks to batch together. Default=100
+- `--max-blocks-batch MAX_BLOCKS_BATCH` Maximum number of blocks to batch together. Default=1000 **Note**: it is used only in `--only-events` mode
+- 
 
 ### `moonworm generate-brownie`:
 

moonworm/cli.py

@@ -19,6 +19,7 @@ from .generators.basic import (
     generate_contract_interface_content,
 )
 from .generators.brownie import generate_brownie_interface
+from .version import MOONWORM_VERSION
 
 
 def write_file(content: str, path: str):
@@ -156,6 +157,10 @@ def handle_watch(args: argparse.Namespace) -> None:
             num_confirmations=args.confirmations,
             start_block=args.start,
             end_block=args.end,
+            min_blocks_batch=args.min_blocks_batch,
+            max_blocks_batch=args.max_blocks_batch,
+            batch_size_update_threshold=args.batch_size_update_threshold,
+            only_events=args.only_events,
             outfile=args.outfile,
         )
 
@@ -200,7 +205,13 @@ def handle_find_deployment(args: argparse.Namespace) -> None:
 
 def generate_argument_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description="Moonworm: Manage your smart contract")
-
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version=f"moonworm {MOONWORM_VERSION}",
+        help="Show version",
+    )
     parser.set_defaults(func=lambda _: parser.print_help())
     subcommands = parser.add_subparsers(dest="subcommands")
 
@@ -265,7 +276,34 @@ def generate_argument_parser() -> argparse.ArgumentParser:
         "--confirmations",
         default=15,
         type=int,
-        help="Number of confirmations to wait for. Default=12",
+        help="Number of confirmations to wait for. Default=15",
+    )
+
+    watch_parser.add_argument(
+        "--min-blocks-batch",
+        default=100,
+        type=int,
+        help="Minimum number of blocks to batch together. Default=100",
+    )
+
+    watch_parser.add_argument(
+        "--max-blocks-batch",
+        default=1000,
+        type=int,
+        help="Maximum number of blocks to batch together. Default=1000",
+    )
+
+    watch_parser.add_argument(
+        "--batch-size-update-threshold",
+        default=100,
+        type=int,
+        help="Number of minimum events  before updating batch size (only for --only-events mode). Default=100",
+    )
+
+    watch_parser.add_argument(
+        "--only-events",
+        action="store_true",
+        help="Only watch events. Default=False",
     )
 
     watch_parser.add_argument(

moonworm/crawler/ethereum_state_provider.py

@@ -81,4 +81,4 @@ class Web3StateProvider(EthereumStateProvider):
         block = self._get_block(block_number)
 
         all_transactions = block["transactions"]
-        return [tx for tx in all_transactions if tx["to"] == address]
+        return [tx for tx in all_transactions if tx.get("to") == address]

moonworm/fixture/abis/DiamondCutFacet.json

@@ -0,0 +1,84 @@
+[
+    {
+        "anonymous": false,
+        "inputs": [
+            {
+                "components": [
+                    {
+                        "internalType": "address",
+                        "name": "facetAddress",
+                        "type": "address"
+                    },
+                    {
+                        "internalType": "enum IDiamondCut.FacetCutAction",
+                        "name": "action",
+                        "type": "uint8"
+                    },
+                    {
+                        "internalType": "bytes4[]",
+                        "name": "functionSelectors",
+                        "type": "bytes4[]"
+                    }
+                ],
+                "indexed": false,
+                "internalType": "struct IDiamondCut.FacetCut[]",
+                "name": "_diamondCut",
+                "type": "tuple[]"
+            },
+            {
+                "indexed": false,
+                "internalType": "address",
+                "name": "_init",
+                "type": "address"
+            },
+            {
+                "indexed": false,
+                "internalType": "bytes",
+                "name": "_calldata",
+                "type": "bytes"
+            }
+        ],
+        "name": "DiamondCut",
+        "type": "event"
+    },
+    {
+        "inputs": [
+            {
+                "components": [
+                    {
+                        "internalType": "address",
+                        "name": "facetAddress",
+                        "type": "address"
+                    },
+                    {
+                        "internalType": "enum IDiamondCut.FacetCutAction",
+                        "name": "action",
+                        "type": "uint8"
+                    },
+                    {
+                        "internalType": "bytes4[]",
+                        "name": "functionSelectors",
+                        "type": "bytes4[]"
+                    }
+                ],
+                "internalType": "struct IDiamondCut.FacetCut[]",
+                "name": "_diamondCut",
+                "type": "tuple[]"
+            },
+            {
+                "internalType": "address",
+                "name": "_init",
+                "type": "address"
+            },
+            {
+                "internalType": "bytes",
+                "name": "_calldata",
+                "type": "bytes"
+            }
+        ],
+        "name": "diamondCut",
+        "outputs": [],
+        "stateMutability": "nonpayable",
+        "type": "function"
+    }
+]

moonworm/version.py

@@ -1 +1 @@
-MOONWORM_VERSION = "0.1.20"
+MOONWORM_VERSION = "0.2.0"

moonworm/watch.py

@@ -2,7 +2,7 @@ import json
 import pprint as pp
 import time
 from dataclasses import asdict
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 from eth_typing.evm import ChecksumAddress
 from tqdm import tqdm
@@ -53,11 +53,49 @@ def watch_contract(
     sleep_time: float = 1,
     start_block: Optional[int] = None,
     end_block: Optional[int] = None,
+    min_blocks_batch: int = 100,
+    max_blocks_batch: int = 5000,
+    batch_size_update_threshold: int = 100,
+    only_events: bool = False,
     outfile: Optional[str] = None,
 ) -> None:
     """
     Watches a contract for events and calls.
     """
+
+    def _crawl_events(
+        event_abi, from_block: int, to_block: int, batch_size: int
+    ) -> Tuple[List[Dict[str, Any]], int]:
+        """
+        Crawls events from the given block range.
+        reduces the batch_size if response is failing.
+        increases the batch_size if response is successful.
+        """
+        events = []
+        current_from_block = from_block
+
+        while current_from_block <= to_block:
+            current_to_block = min(current_from_block + batch_size, to_block)
+            try:
+                events_chunk = _fetch_events_chunk(
+                    web3,
+                    event_abi,
+                    current_from_block,
+                    current_to_block,
+                    [contract_address],
+                )
+                events.extend(events_chunk)
+                current_from_block = current_to_block + 1
+                if len(events) <= batch_size_update_threshold:
+                    batch_size = min(batch_size * 2, max_blocks_batch)
+            except Exception as e:
+                if batch_size <= min_blocks_batch:
+                    raise e
+                time.sleep(0.1)
+                batch_size = max(batch_size // 2, min_blocks_batch)
+        return events, batch_size
+
+    current_batch_size = min_blocks_batch
     state = MockState()
     crawler = FunctionCallCrawler(
         state,
@@ -83,7 +121,8 @@ def watch_contract(
         while end_block is None or current_block <= end_block:
             time.sleep(sleep_time)
             until_block = min(
-                web3.eth.blockNumber - num_confirmations, current_block + 100
+                web3.eth.blockNumber - num_confirmations,
+                current_block + current_batch_size,
             )
             if end_block is not None:
                 until_block = min(until_block, end_block)
@@ -92,25 +131,26 @@ def watch_contract(
                 continue
 
             sleep_time /= 2
-
+            if not only_events:
-            crawler.crawl(current_block, until_block)
+                crawler.crawl(current_block, until_block)
-            if state.state:
+                if state.state:
-                print("Got transaction calls:")
+                    print("Got transaction calls:")
-                for call in state.state:
+                    for call in state.state:
-                    pp.pprint(call, width=200, indent=4)
+                        pp.pprint(call, width=200, indent=4)
-                    if ofp is not None:
+                        if ofp is not None:
-                        print(json.dumps(asdict(call)), file=ofp)
+                            print(json.dumps(asdict(call)), file=ofp)
-                        ofp.flush()
+                            ofp.flush()
-                state.flush()
+                    state.flush()
 
             for event_abi in event_abis:
-                all_events = _fetch_events_chunk(
+                all_events, new_batch_size = _crawl_events(
-                    web3,
+                    event_abi, current_block, until_block, current_batch_size
-                    event_abi,
-                    current_block,
-                    until_block,
-                    [contract_address],
                 )
+
+                if only_events:
+                    # Updating batch size only in `--only-events` mode
+                    # otherwise it will start taking too much if we also crawl transactions
+                    current_batch_size = new_batch_size
                 for event in all_events:
                     print("Got event:")
                     pp.pprint(event, width=200, indent=4)

setup.py

@@ -19,7 +19,7 @@ setup(
         "pysha3<2.0.0,>=1.0.0",
         "tqdm",
         "typing-extensions<4,>=3.7.4",
-        "web3[tester]",
+        "web3[tester] >=5.29.0",
     ],
     extras_require={
         "dev": [