From 405755215f5c9ae05066809380103a32cc763844 Mon Sep 17 00:00:00 2001 From: Yhtyyar Sahatov Date: Tue, 19 Apr 2022 20:25:20 +0300 Subject: [PATCH 1/6] added --- moonworm/cli.py | 33 +++++++- moonworm/crawler/ethereum_state_provider.py | 2 +- moonworm/crawler/log_scanner.py | 2 + moonworm/fixture/abis/DiamondCutFacet.json | 84 +++++++++++++++++++++ moonworm/watch.py | 83 +++++++++++++++----- setup.py | 2 +- 6 files changed, 185 insertions(+), 21 deletions(-) create mode 100644 moonworm/fixture/abis/DiamondCutFacet.json diff --git a/moonworm/cli.py b/moonworm/cli.py index 545f2cc..10621cc 100644 --- a/moonworm/cli.py +++ b/moonworm/cli.py @@ -156,6 +156,10 @@ def handle_watch(args: argparse.Namespace) -> None: num_confirmations=args.confirmations, start_block=args.start, end_block=args.end, + min_blocks_batch=args.min_blocks_batch, + max_blocks_batch=args.max_blocks_batch, + batch_size_update_threshold=args.batch_size_update_threshold, + only_events=args.only_events, outfile=args.outfile, ) @@ -265,7 +269,34 @@ def generate_argument_parser() -> argparse.ArgumentParser: "--confirmations", default=15, type=int, - help="Number of confirmations to wait for. Default=12", + help="Number of confirmations to wait for. Default=15", + ) + + watch_parser.add_argument( + "--min-blocks-batch", + default=100, + type=int, + help="Minimum number of blocks to batch together. Default=100", + ) + + watch_parser.add_argument( + "--max-blocks-batch", + default=1000, + type=int, + help="Maximum number of blocks to batch together. Default=1000", + ) + + watch_parser.add_argument( + "--batch-size-update-threshold", + default=100, + type=int, + help="Number of minimum events before updating batch size (only for --only-events mode). Default=100", + ) + + watch_parser.add_argument( + "--only-events", + action="store_true", + help="Only watch events. Default=False", ) watch_parser.add_argument( diff --git a/moonworm/crawler/ethereum_state_provider.py b/moonworm/crawler/ethereum_state_provider.py index f801719..c584074 100644 --- a/moonworm/crawler/ethereum_state_provider.py +++ b/moonworm/crawler/ethereum_state_provider.py @@ -81,4 +81,4 @@ class Web3StateProvider(EthereumStateProvider): block = self._get_block(block_number) all_transactions = block["transactions"] - return [tx for tx in all_transactions if tx["to"] == address] + return [tx for tx in all_transactions if tx.get("to") == address] diff --git a/moonworm/crawler/log_scanner.py b/moonworm/crawler/log_scanner.py index c07c69c..db98f31 100644 --- a/moonworm/crawler/log_scanner.py +++ b/moonworm/crawler/log_scanner.py @@ -121,6 +121,8 @@ def _fetch_events_chunk( } all_events.append(event) except Exception as e: + # TODO remove: + print(log) if on_decode_error: on_decode_error(e) continue diff --git a/moonworm/fixture/abis/DiamondCutFacet.json b/moonworm/fixture/abis/DiamondCutFacet.json new file mode 100644 index 0000000..67311ed --- /dev/null +++ b/moonworm/fixture/abis/DiamondCutFacet.json @@ -0,0 +1,84 @@ +[ + { + "anonymous": false, + "inputs": [ + { + "components": [ + { + "internalType": "address", + "name": "facetAddress", + "type": "address" + }, + { + "internalType": "enum IDiamondCut.FacetCutAction", + "name": "action", + "type": "uint8" + }, + { + "internalType": "bytes4[]", + "name": "functionSelectors", + "type": "bytes4[]" + } + ], + "indexed": false, + "internalType": "struct IDiamondCut.FacetCut[]", + "name": "_diamondCut", + "type": "tuple[]" + }, + { + "indexed": false, + "internalType": "address", + "name": "_init", + "type": "address" + }, + { + "indexed": false, + "internalType": "bytes", + "name": "_calldata", + "type": "bytes" + } + ], + "name": "DiamondCut", + "type": "event" + }, + { + "inputs": [ + { + "components": [ + { + "internalType": "address", + "name": "facetAddress", + "type": "address" + }, + { + "internalType": "enum IDiamondCut.FacetCutAction", + "name": "action", + "type": "uint8" + }, + { + "internalType": "bytes4[]", + "name": "functionSelectors", + "type": "bytes4[]" + } + ], + "internalType": "struct IDiamondCut.FacetCut[]", + "name": "_diamondCut", + "type": "tuple[]" + }, + { + "internalType": "address", + "name": "_init", + "type": "address" + }, + { + "internalType": "bytes", + "name": "_calldata", + "type": "bytes" + } + ], + "name": "diamondCut", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } +] \ No newline at end of file diff --git a/moonworm/watch.py b/moonworm/watch.py index 3504982..88963b9 100644 --- a/moonworm/watch.py +++ b/moonworm/watch.py @@ -2,7 +2,7 @@ import json import pprint as pp import time from dataclasses import asdict -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from eth_typing.evm import ChecksumAddress from tqdm import tqdm @@ -53,11 +53,51 @@ def watch_contract( sleep_time: float = 1, start_block: Optional[int] = None, end_block: Optional[int] = None, + min_blocks_batch: int = 100, + max_blocks_batch: int = 5000, + batch_size_update_threshold: int = 100, + only_events: bool = False, outfile: Optional[str] = None, ) -> None: """ Watches a contract for events and calls. """ + + def _crawl_events( + event_abi, from_block: int, to_block: int, batch_size: int + ) -> Tuple[List[Dict[str, Any]], int]: + """ + Crawls events from the given block range. + reduces the batch_size if response is failing. + increases the batch_size if response is successful. + """ + events = [] + current_from_block = from_block + + while current_from_block <= to_block: + current_to_block = min(current_from_block + batch_size, to_block) + print(f"From block: {current_from_block}, to block: {current_to_block}") + try: + events_chunk = _fetch_events_chunk( + web3, + event_abi, + current_from_block, + current_to_block, + [contract_address], + on_decode_error=lambda e: print(e), + ) + events.extend(events_chunk) + current_from_block = current_to_block + 1 + if len(events) <= batch_size_update_threshold: + batch_size = min(batch_size * 2, max_blocks_batch) + except Exception as e: + if batch_size <= min_blocks_batch: + raise e + time.sleep(0.1) + batch_size = max(batch_size // 2, min_blocks_batch) + return events, batch_size + + current_batch_size = min_blocks_batch state = MockState() crawler = FunctionCallCrawler( state, @@ -83,7 +123,8 @@ def watch_contract( while end_block is None or current_block <= end_block: time.sleep(sleep_time) until_block = min( - web3.eth.blockNumber - num_confirmations, current_block + 100 + web3.eth.blockNumber - num_confirmations, + current_block + current_batch_size, ) if end_block is not None: until_block = min(until_block, end_block) @@ -92,25 +133,31 @@ def watch_contract( continue sleep_time /= 2 - - crawler.crawl(current_block, until_block) - if state.state: - print("Got transaction calls:") - for call in state.state: - pp.pprint(call, width=200, indent=4) - if ofp is not None: - print(json.dumps(asdict(call)), file=ofp) - ofp.flush() - state.flush() + if not only_events: + crawler.crawl(current_block, until_block) + if state.state: + print("Got transaction calls:") + for call in state.state: + pp.pprint(call, width=200, indent=4) + if ofp is not None: + print(json.dumps(asdict(call)), file=ofp) + ofp.flush() + state.flush() for event_abi in event_abis: - all_events = _fetch_events_chunk( - web3, - event_abi, - current_block, - until_block, - [contract_address], + all_events, new_batch_size = _crawl_events( + event_abi, current_block, until_block, current_batch_size ) + # TODO: remove + if all_events: + print("Got events:") + else: + print("No events") + + if only_events: + # Updating batch size only in `--only-events` mode + # otherwise it will start taking too much if we also crawl transactions + current_batch_size = new_batch_size for event in all_events: print("Got event:") pp.pprint(event, width=200, indent=4) diff --git a/setup.py b/setup.py index ad988cb..d5f1955 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( "pysha3<2.0.0,>=1.0.0", "tqdm", "typing-extensions<4,>=3.7.4", - "web3[tester]", + "web3[tester] >=5.29.0", ], extras_require={ "dev": [ From 2f63ef1910b2a9d194f828abcef434a5d7279d34 Mon Sep 17 00:00:00 2001 From: Yhtyyar Sahatov Date: Tue, 19 Apr 2022 21:45:57 +0300 Subject: [PATCH 2/6] small fixes --- moonworm/crawler/log_scanner.py | 2 -- moonworm/version.py | 2 +- moonworm/watch.py | 7 ------- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/moonworm/crawler/log_scanner.py b/moonworm/crawler/log_scanner.py index db98f31..c07c69c 100644 --- a/moonworm/crawler/log_scanner.py +++ b/moonworm/crawler/log_scanner.py @@ -121,8 +121,6 @@ def _fetch_events_chunk( } all_events.append(event) except Exception as e: - # TODO remove: - print(log) if on_decode_error: on_decode_error(e) continue diff --git a/moonworm/version.py b/moonworm/version.py index aa25077..745e5c4 100644 --- a/moonworm/version.py +++ b/moonworm/version.py @@ -1 +1 @@ -MOONWORM_VERSION = "0.1.20" +MOONWORM_VERSION = "0.2.0" diff --git a/moonworm/watch.py b/moonworm/watch.py index 88963b9..c08887f 100644 --- a/moonworm/watch.py +++ b/moonworm/watch.py @@ -76,7 +76,6 @@ def watch_contract( while current_from_block <= to_block: current_to_block = min(current_from_block + batch_size, to_block) - print(f"From block: {current_from_block}, to block: {current_to_block}") try: events_chunk = _fetch_events_chunk( web3, @@ -84,7 +83,6 @@ def watch_contract( current_from_block, current_to_block, [contract_address], - on_decode_error=lambda e: print(e), ) events.extend(events_chunk) current_from_block = current_to_block + 1 @@ -148,11 +146,6 @@ def watch_contract( all_events, new_batch_size = _crawl_events( event_abi, current_block, until_block, current_batch_size ) - # TODO: remove - if all_events: - print("Got events:") - else: - print("No events") if only_events: # Updating batch size only in `--only-events` mode From 2d82c43ce9f8437df01bf7f3d39d7219766ebe60 Mon Sep 17 00:00:00 2001 From: Yhtyyar Sahatov Date: Tue, 19 Apr 2022 21:56:45 +0300 Subject: [PATCH 3/6] added `--version` --- moonworm/cli.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/moonworm/cli.py b/moonworm/cli.py index 10621cc..2884f7e 100644 --- a/moonworm/cli.py +++ b/moonworm/cli.py @@ -19,6 +19,7 @@ from .generators.basic import ( generate_contract_interface_content, ) from .generators.brownie import generate_brownie_interface +from .version import MOONWORM_VERSION def write_file(content: str, path: str): @@ -202,9 +203,18 @@ def handle_find_deployment(args: argparse.Namespace) -> None: print(result) +# def handle_main_parset(args:argparse.Namespace) -> None: + + def generate_argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Moonworm: Manage your smart contract") - + parser.add_argument( + "-v", + "--version", + action="version", + version=f"moonworm {MOONWORM_VERSION}", + help="Show version", + ) parser.set_defaults(func=lambda _: parser.print_help()) subcommands = parser.add_subparsers(dest="subcommands") From 85aaa94b266bd9f06becec18c9fc9fbd8f63de4f Mon Sep 17 00:00:00 2001 From: Yhtyyar Sahatov Date: Tue, 19 Apr 2022 22:02:28 +0300 Subject: [PATCH 4/6] update readme --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9a781de..6ce158a 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Moonworm is a set of tools that helps you develop/analyze blockchain dapps. Pump ## Setup: -```sql +```bash pip install moonworm ``` @@ -33,15 +33,18 @@ Arguments: - `--contract/-c CONTRACT` Contract address - `--web3/-w WEB3` Web3 provider uri - `--start/-s START` block to start watching -- `--end/-e END` block to stop crawling, if not given, crawler will not stop Optional args: - +- `--end/-e END` block to stop crawling, if not given, crawler will not stop - `--poa` Flag for `PoA` networks, for example `polygon` - `--confirmations CONFIRMATIONS` Number of confirmations to set for watch. (Default 12) - `--outfile/-o OUTFILE` `JSONL` file into which to write events and transactions - `--db` Use Moonstream database specified by `MOONSTREAM_DB_URI` to get blocks/transactions. If set, need also provide `--network` - `-network {ethereum,polygon}`Network name that represents models from db. If the `--db` is set, required +- `--only-events` Flag, if set: only watches events. Default=`False` +- `--min-blocks-batch MIN_BLOCKS_BATCH` Minimum number of blocks to batch together. Default=100 +- `--max-blocks-batch MAX_BLOCKS_BATCH` Maximum number of blocks to batch together. Default=1000 **Note**: it is used only for `--only-events` mode +- ### `moonworm generate-brownie`: From eaa898763a00099f72ff3264df7b57981e97d1c2 Mon Sep 17 00:00:00 2001 From: Yhtyyar Sahatov Date: Tue, 19 Apr 2022 22:06:44 +0300 Subject: [PATCH 5/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ce158a..c644908 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Optional args: - `-network {ethereum,polygon}`Network name that represents models from db. If the `--db` is set, required - `--only-events` Flag, if set: only watches events. Default=`False` - `--min-blocks-batch MIN_BLOCKS_BATCH` Minimum number of blocks to batch together. Default=100 -- `--max-blocks-batch MAX_BLOCKS_BATCH` Maximum number of blocks to batch together. Default=1000 **Note**: it is used only for `--only-events` mode +- `--max-blocks-batch MAX_BLOCKS_BATCH` Maximum number of blocks to batch together. Default=1000 **Note**: it is used only in `--only-events` mode - ### `moonworm generate-brownie`: From 794d2107ee7395578dcafffb782de82e563e1dac Mon Sep 17 00:00:00 2001 From: Yhtyyar Sahatov Date: Tue, 19 Apr 2022 22:07:29 +0300 Subject: [PATCH 6/6] removed comment --- moonworm/cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/moonworm/cli.py b/moonworm/cli.py index 2884f7e..c009a63 100644 --- a/moonworm/cli.py +++ b/moonworm/cli.py @@ -203,9 +203,6 @@ def handle_find_deployment(args: argparse.Namespace) -> None: print(result) -# def handle_main_parset(args:argparse.Namespace) -> None: - - def generate_argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Moonworm: Manage your smart contract") parser.add_argument(