diff --git a/README.md b/README.md index 9a781de..c644908 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Moonworm is a set of tools that helps you develop/analyze blockchain dapps. Pump ## Setup: -```sql +```bash pip install moonworm ``` @@ -33,15 +33,18 @@ Arguments: - `--contract/-c CONTRACT` Contract address - `--web3/-w WEB3` Web3 provider uri - `--start/-s START` block to start watching -- `--end/-e END` block to stop crawling, if not given, crawler will not stop Optional args: - +- `--end/-e END` block to stop crawling, if not given, crawler will not stop - `--poa` Flag for `PoA` networks, for example `polygon` - `--confirmations CONFIRMATIONS` Number of confirmations to set for watch. (Default 12) - `--outfile/-o OUTFILE` `JSONL` file into which to write events and transactions - `--db` Use Moonstream database specified by `MOONSTREAM_DB_URI` to get blocks/transactions. If set, need also provide `--network` - `-network {ethereum,polygon}`Network name that represents models from db. If the `--db` is set, required +- `--only-events` Flag, if set: only watches events. Default=`False` +- `--min-blocks-batch MIN_BLOCKS_BATCH` Minimum number of blocks to batch together. Default=100 +- `--max-blocks-batch MAX_BLOCKS_BATCH` Maximum number of blocks to batch together. Default=1000 **Note**: it is used only in `--only-events` mode +- ### `moonworm generate-brownie`: diff --git a/moonworm/cli.py b/moonworm/cli.py index 545f2cc..c009a63 100644 --- a/moonworm/cli.py +++ b/moonworm/cli.py @@ -19,6 +19,7 @@ from .generators.basic import ( generate_contract_interface_content, ) from .generators.brownie import generate_brownie_interface +from .version import MOONWORM_VERSION def write_file(content: str, path: str): @@ -156,6 +157,10 @@ def handle_watch(args: argparse.Namespace) -> None: num_confirmations=args.confirmations, start_block=args.start, end_block=args.end, + min_blocks_batch=args.min_blocks_batch, + max_blocks_batch=args.max_blocks_batch, + batch_size_update_threshold=args.batch_size_update_threshold, + only_events=args.only_events, outfile=args.outfile, ) @@ -200,7 +205,13 @@ def handle_find_deployment(args: argparse.Namespace) -> None: def generate_argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Moonworm: Manage your smart contract") - + parser.add_argument( + "-v", + "--version", + action="version", + version=f"moonworm {MOONWORM_VERSION}", + help="Show version", + ) parser.set_defaults(func=lambda _: parser.print_help()) subcommands = parser.add_subparsers(dest="subcommands") @@ -265,7 +276,34 @@ def generate_argument_parser() -> argparse.ArgumentParser: "--confirmations", default=15, type=int, - help="Number of confirmations to wait for. Default=12", + help="Number of confirmations to wait for. Default=15", + ) + + watch_parser.add_argument( + "--min-blocks-batch", + default=100, + type=int, + help="Minimum number of blocks to batch together. Default=100", + ) + + watch_parser.add_argument( + "--max-blocks-batch", + default=1000, + type=int, + help="Maximum number of blocks to batch together. Default=1000", + ) + + watch_parser.add_argument( + "--batch-size-update-threshold", + default=100, + type=int, + help="Number of minimum events before updating batch size (only for --only-events mode). Default=100", + ) + + watch_parser.add_argument( + "--only-events", + action="store_true", + help="Only watch events. Default=False", ) watch_parser.add_argument( diff --git a/moonworm/crawler/ethereum_state_provider.py b/moonworm/crawler/ethereum_state_provider.py index f801719..c584074 100644 --- a/moonworm/crawler/ethereum_state_provider.py +++ b/moonworm/crawler/ethereum_state_provider.py @@ -81,4 +81,4 @@ class Web3StateProvider(EthereumStateProvider): block = self._get_block(block_number) all_transactions = block["transactions"] - return [tx for tx in all_transactions if tx["to"] == address] + return [tx for tx in all_transactions if tx.get("to") == address] diff --git a/moonworm/fixture/abis/DiamondCutFacet.json b/moonworm/fixture/abis/DiamondCutFacet.json new file mode 100644 index 0000000..67311ed --- /dev/null +++ b/moonworm/fixture/abis/DiamondCutFacet.json @@ -0,0 +1,84 @@ +[ + { + "anonymous": false, + "inputs": [ + { + "components": [ + { + "internalType": "address", + "name": "facetAddress", + "type": "address" + }, + { + "internalType": "enum IDiamondCut.FacetCutAction", + "name": "action", + "type": "uint8" + }, + { + "internalType": "bytes4[]", + "name": "functionSelectors", + "type": "bytes4[]" + } + ], + "indexed": false, + "internalType": "struct IDiamondCut.FacetCut[]", + "name": "_diamondCut", + "type": "tuple[]" + }, + { + "indexed": false, + "internalType": "address", + "name": "_init", + "type": "address" + }, + { + "indexed": false, + "internalType": "bytes", + "name": "_calldata", + "type": "bytes" + } + ], + "name": "DiamondCut", + "type": "event" + }, + { + "inputs": [ + { + "components": [ + { + "internalType": "address", + "name": "facetAddress", + "type": "address" + }, + { + "internalType": "enum IDiamondCut.FacetCutAction", + "name": "action", + "type": "uint8" + }, + { + "internalType": "bytes4[]", + "name": "functionSelectors", + "type": "bytes4[]" + } + ], + "internalType": "struct IDiamondCut.FacetCut[]", + "name": "_diamondCut", + "type": "tuple[]" + }, + { + "internalType": "address", + "name": "_init", + "type": "address" + }, + { + "internalType": "bytes", + "name": "_calldata", + "type": "bytes" + } + ], + "name": "diamondCut", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } +] \ No newline at end of file diff --git a/moonworm/version.py b/moonworm/version.py index aa25077..745e5c4 100644 --- a/moonworm/version.py +++ b/moonworm/version.py @@ -1 +1 @@ -MOONWORM_VERSION = "0.1.20" +MOONWORM_VERSION = "0.2.0" diff --git a/moonworm/watch.py b/moonworm/watch.py index 3504982..c08887f 100644 --- a/moonworm/watch.py +++ b/moonworm/watch.py @@ -2,7 +2,7 @@ import json import pprint as pp import time from dataclasses import asdict -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from eth_typing.evm import ChecksumAddress from tqdm import tqdm @@ -53,11 +53,49 @@ def watch_contract( sleep_time: float = 1, start_block: Optional[int] = None, end_block: Optional[int] = None, + min_blocks_batch: int = 100, + max_blocks_batch: int = 5000, + batch_size_update_threshold: int = 100, + only_events: bool = False, outfile: Optional[str] = None, ) -> None: """ Watches a contract for events and calls. """ + + def _crawl_events( + event_abi, from_block: int, to_block: int, batch_size: int + ) -> Tuple[List[Dict[str, Any]], int]: + """ + Crawls events from the given block range. + reduces the batch_size if response is failing. + increases the batch_size if response is successful. + """ + events = [] + current_from_block = from_block + + while current_from_block <= to_block: + current_to_block = min(current_from_block + batch_size, to_block) + try: + events_chunk = _fetch_events_chunk( + web3, + event_abi, + current_from_block, + current_to_block, + [contract_address], + ) + events.extend(events_chunk) + current_from_block = current_to_block + 1 + if len(events) <= batch_size_update_threshold: + batch_size = min(batch_size * 2, max_blocks_batch) + except Exception as e: + if batch_size <= min_blocks_batch: + raise e + time.sleep(0.1) + batch_size = max(batch_size // 2, min_blocks_batch) + return events, batch_size + + current_batch_size = min_blocks_batch state = MockState() crawler = FunctionCallCrawler( state, @@ -83,7 +121,8 @@ def watch_contract( while end_block is None or current_block <= end_block: time.sleep(sleep_time) until_block = min( - web3.eth.blockNumber - num_confirmations, current_block + 100 + web3.eth.blockNumber - num_confirmations, + current_block + current_batch_size, ) if end_block is not None: until_block = min(until_block, end_block) @@ -92,25 +131,26 @@ def watch_contract( continue sleep_time /= 2 - - crawler.crawl(current_block, until_block) - if state.state: - print("Got transaction calls:") - for call in state.state: - pp.pprint(call, width=200, indent=4) - if ofp is not None: - print(json.dumps(asdict(call)), file=ofp) - ofp.flush() - state.flush() + if not only_events: + crawler.crawl(current_block, until_block) + if state.state: + print("Got transaction calls:") + for call in state.state: + pp.pprint(call, width=200, indent=4) + if ofp is not None: + print(json.dumps(asdict(call)), file=ofp) + ofp.flush() + state.flush() for event_abi in event_abis: - all_events = _fetch_events_chunk( - web3, - event_abi, - current_block, - until_block, - [contract_address], + all_events, new_batch_size = _crawl_events( + event_abi, current_block, until_block, current_batch_size ) + + if only_events: + # Updating batch size only in `--only-events` mode + # otherwise it will start taking too much if we also crawl transactions + current_batch_size = new_batch_size for event in all_events: print("Got event:") pp.pprint(event, width=200, indent=4) diff --git a/setup.py b/setup.py index ad988cb..d5f1955 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( "pysha3<2.0.0,>=1.0.0", "tqdm", "typing-extensions<4,>=3.7.4", - "web3[tester]", + "web3[tester] >=5.29.0", ], extras_require={ "dev": [