From 09abceab4d4fea110bfa1f3571cae908f08bb7de Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Mon, 28 Feb 2022 15:00:54 -0800 Subject: [PATCH 1/3] Added --outfile argument to "moonworm watch" If specified, in addition to printing events and method calls to screen, they are appended to the given file as JSON lines. --- moonworm/cli.py | 8 +++++ moonworm/version.py | 2 +- moonworm/watch.py | 74 +++++++++++++++++++++++++++------------------ 3 files changed, 53 insertions(+), 31 deletions(-) diff --git a/moonworm/cli.py b/moonworm/cli.py index 0e6a146..fa971dc 100644 --- a/moonworm/cli.py +++ b/moonworm/cli.py @@ -138,6 +138,7 @@ def handle_watch(args: argparse.Namespace) -> None: contract_abi=contract_abi, num_confirmations=args.confirmations, start_block=args.start, + outfile=args.outfile, ) finally: state_provider.clear_db_session() @@ -244,6 +245,13 @@ def generate_argument_parser() -> argparse.ArgumentParser: help="Number of confirmations to wait for. Default=12", ) + watch_parser.add_argument( + "-o", + "--outfile", + default=None, + help="Optional JSONL (JsON lines) file into which to write events and method calls", + ) + watch_parser.set_defaults(func=handle_watch) watch_cu_parser = subcommands.add_parser( diff --git a/moonworm/version.py b/moonworm/version.py index beb9964..6a20685 100644 --- a/moonworm/version.py +++ b/moonworm/version.py @@ -1 +1 @@ -MOONWORM_VERSION = "0.1.16" +MOONWORM_VERSION = "0.1.17" diff --git a/moonworm/watch.py b/moonworm/watch.py index 124ed98..73027e3 100644 --- a/moonworm/watch.py +++ b/moonworm/watch.py @@ -1,8 +1,9 @@ +from dataclasses import asdict +import json import pprint as pp import time from typing import Any, Dict, List, Optional -import web3 from eth_typing.evm import ChecksumAddress from tqdm import tqdm from web3 import Web3 @@ -51,6 +52,7 @@ def watch_contract( num_confirmations: int = 10, sleep_time: float = 1, start_block: Optional[int] = None, + outfile: Optional[str] = None, ) -> None: """ Watches a contract for events and calls. @@ -72,34 +74,46 @@ def watch_contract( progress_bar = tqdm(unit=" blocks") progress_bar.set_description(f"Current block {current_block}") - while True: - time.sleep(sleep_time) - end_block = min(web3.eth.blockNumber - num_confirmations, current_block + 100) - if end_block < current_block: - sleep_time *= 2 - continue - - sleep_time /= 2 - - crawler.crawl(current_block, end_block) - if state.state: - print("Got transaction calls:") - for call in state.state: - pp.pprint(call, width=200, indent=4) - state.flush() - - for event_abi in event_abis: - all_events = _fetch_events_chunk( - web3, - event_abi, - current_block, - end_block, - [contract_address], + ofp = None + if outfile is not None: + ofp = open(outfile, "a") + try: + while True: + time.sleep(sleep_time) + end_block = min( + web3.eth.blockNumber - num_confirmations, current_block + 100 ) - for event in all_events: - print("Got event:") - pp.pprint(event, width=200, indent=4) + if end_block < current_block: + sleep_time *= 2 + continue - progress_bar.set_description(f"Current block {end_block}, Already watching for") - progress_bar.update(end_block - current_block + 1) - current_block = end_block + 1 + sleep_time /= 2 + + crawler.crawl(current_block, end_block) + if state.state: + print("Got transaction calls:") + for call in state.state: + pp.pprint(call, width=200, indent=4) + json.dump(asdict(call), ofp) + state.flush() + + for event_abi in event_abis: + all_events = _fetch_events_chunk( + web3, + event_abi, + current_block, + end_block, + [contract_address], + ) + for event in all_events: + print("Got event:") + pp.pprint(event, width=200, indent=4) + json.dump(event, ofp) + + progress_bar.set_description( + f"Current block {end_block}, Already watching for" + ) + progress_bar.update(end_block - current_block + 1) + current_block = end_block + 1 + finally: + ofp.close() From 042aca21fb2eb8957d545f78dfe85440dbce0ea6 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Mon, 28 Feb 2022 15:18:53 -0800 Subject: [PATCH 2/3] Fixes and improvements There are two places where `watch_contract` is called from CLI, and I only modified one of them before. Also used `print(json.dumps(...), file=ofp)` pattern to write with newline. --- moonworm/cli.py | 1 + moonworm/watch.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/moonworm/cli.py b/moonworm/cli.py index fa971dc..8cf7c12 100644 --- a/moonworm/cli.py +++ b/moonworm/cli.py @@ -152,6 +152,7 @@ def handle_watch(args: argparse.Namespace) -> None: contract_abi=contract_abi, num_confirmations=args.confirmations, start_block=args.start, + outfile=args.outfile, ) diff --git a/moonworm/watch.py b/moonworm/watch.py index 73027e3..e7fafb0 100644 --- a/moonworm/watch.py +++ b/moonworm/watch.py @@ -94,7 +94,8 @@ def watch_contract( print("Got transaction calls:") for call in state.state: pp.pprint(call, width=200, indent=4) - json.dump(asdict(call), ofp) + if ofp is not None: + print(json.dumps(asdict(call)), file=ofp) state.flush() for event_abi in event_abis: @@ -108,7 +109,8 @@ def watch_contract( for event in all_events: print("Got event:") pp.pprint(event, width=200, indent=4) - json.dump(event, ofp) + if ofp is not None: + print(json.dumps(event), file=ofp) progress_bar.set_description( f"Current block {end_block}, Already watching for" @@ -116,4 +118,5 @@ def watch_contract( progress_bar.update(end_block - current_block + 1) current_block = end_block + 1 finally: - ofp.close() + if ofp is not None: + ofp.close() From b64b36b3c23dbfe17bb57f1ce767ae1143d0af13 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Mon, 28 Feb 2022 16:07:07 -0800 Subject: [PATCH 3/3] Fixed isort, added ofp.flush() --- moonworm/watch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moonworm/watch.py b/moonworm/watch.py index e7fafb0..d77ff1d 100644 --- a/moonworm/watch.py +++ b/moonworm/watch.py @@ -1,7 +1,7 @@ -from dataclasses import asdict import json import pprint as pp import time +from dataclasses import asdict from typing import Any, Dict, List, Optional from eth_typing.evm import ChecksumAddress @@ -96,6 +96,7 @@ def watch_contract( pp.pprint(call, width=200, indent=4) if ofp is not None: print(json.dumps(asdict(call)), file=ofp) + ofp.flush() state.flush() for event_abi in event_abis: @@ -111,6 +112,7 @@ def watch_contract( pp.pprint(event, width=200, indent=4) if ofp is not None: print(json.dumps(event), file=ofp) + ofp.flush() progress_bar.set_description( f"Current block {end_block}, Already watching for"