diff --git a/moonworm/cli.py b/moonworm/cli.py index d7f329e..ff4246e 100644 --- a/moonworm/cli.py +++ b/moonworm/cli.py @@ -6,6 +6,7 @@ from shutil import copyfile from web3.main import Web3 from web3.middleware import geth_poa_middleware +from moonworm.crawler.ethereum_state_provider import Web3StateProvider from moonworm.watch import watch_contract @@ -14,6 +15,7 @@ from .generator import ( generate_contract_cli_content, generate_contract_interface_content, ) +from .crawler.networks import Network def write_file(content: str, path: str): @@ -92,9 +94,41 @@ def handle_watch(args: argparse.Namespace) -> None: web3 = Web3(Web3.HTTPProvider(args.web3)) if args.poa: web3.middleware_onion.inject(geth_poa_middleware, layer=0) - watch_contract( - web3, web3.toChecksumAddress(args.contract), contract_abi, args.confirmations - ) + if args.db: + if args.network is None: + raise ValueError("Please specify --network") + network = Network.__members__[args.network] + from .crawler.moonstream_ethereum_state_provider import ( + MoonstreamEthereumStateProvider, + ) + from moonstreamdb.db import yield_db_session_ctx + + state_provider = MoonstreamEthereumStateProvider(web3, network) + + with yield_db_session_ctx() as db_session: + try: + state_provider.set_db_session(db_session) + watch_contract( + web3=web3, + state_provider=state_provider, + contract_address=web3.toChecksumAddress(args.contract), + contract_abi=contract_abi, + num_confirmations=args.confirmations, + start_block=args.start, + ) + finally: + state_provider.clear_db_session() + + else: + state_provider = Web3StateProvider(web3) + watch_contract( + web3=web3, + state_provider=state_provider, + contract_address=web3.toChecksumAddress(args.contract), + contract_abi=contract_abi, + num_confirmations=args.confirmations, + start_block=args.start, + ) def handle_watch_cu(args: argparse.Namespace) -> None: @@ -152,6 +186,27 @@ def generate_argument_parser() -> argparse.ArgumentParser: help="Web3 provider", ) + watch_parser.add_argument( + "--db", + action="store_true", + help="Use Moonstream database specified by 'MOONSTREAM_DB_URI' to get blocks/transactions. If set, need also provide --network", + ) + + watch_parser.add_argument( + "--network", + choices=Network.__members__, + default=None, + help="Network name that represents models from db. If --db is set, required", + ) + + watch_parser.add_argument( + "--start", + "-s", + type=int, + default=None, + help="Block number to start watching from", + ) + watch_parser.add_argument( "--poa", action="store_true", diff --git a/moonworm/crawler/ethereum_state_provider.py b/moonworm/crawler/ethereum_state_provider.py new file mode 100644 index 0000000..1ab9102 --- /dev/null +++ b/moonworm/crawler/ethereum_state_provider.py @@ -0,0 +1,82 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional, Union +import logging +from eth_typing.evm import ChecksumAddress + + +from web3 import Web3 + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class EthereumStateProvider(ABC): + """ + Abstract class for Ethereum state provider. + If you want to use a different state provider, you can implement this class. + """ + + @abstractmethod + def get_last_block_number(self) -> int: + """ + Returns the last block number. + """ + pass + + @abstractmethod + def get_block_timestamp(self, block_number: int) -> int: + """ + Returns the timestamp of the block with the given block number. + """ + pass + + @abstractmethod + def get_transactions_to_address( + self, address, block_number: int + ) -> List[Dict[str, Any]]: + """ + Returns all transactions to the given address in the given block number. + """ + pass + + +class Web3StateProvider(EthereumStateProvider): + """ + Implementation of EthereumStateProvider with web3. + """ + + def __init__(self, w3: Web3): + self.w3 = w3 + + self.blocks_cache = {} + + def get_transaction_reciept(self, transaction_hash: str) -> Dict[str, Any]: + return self.w3.eth.get_transaction_receipt(transaction_hash) + + def get_last_block_number(self) -> int: + return self.w3.eth.block_number + + def _get_block(self, block_number: int) -> Dict[str, Any]: + if block_number in self.blocks_cache: + return self.blocks_cache[block_number] + block = self.w3.eth.getBlock(block_number, full_transactions=True) + + # clear cache if it grows too large + if len(self.blocks_cache) > 50: + self.blocks_cache = {} + + self.blocks_cache[block_number] = block + return block + + def get_block_timestamp(self, block_number: int) -> int: + block = self._get_block(block_number) + return block["timestamp"] + + def get_transactions_to_address( + self, address: ChecksumAddress, block_number: int + ) -> List[Dict[str, Any]]: + block = self._get_block(block_number) + + all_transactions = block["transactions"] + return [tx for tx in all_transactions if tx["to"] == address] diff --git a/moonworm/crawler/function_call_crawler.py b/moonworm/crawler/function_call_crawler.py index c63a5b0..7922c06 100644 --- a/moonworm/crawler/function_call_crawler.py +++ b/moonworm/crawler/function_call_crawler.py @@ -11,6 +11,8 @@ from web3.types import ABI from moonworm.contracts import ERC1155 +from .ethereum_state_provider import EthereumStateProvider, Web3StateProvider + @dataclass class ContractFunctionCall: @@ -25,36 +27,6 @@ class ContractFunctionCall: status: int -class EthereumStateProvider(ABC): - """ - Abstract class for Ethereum state provider. - If you want to use a different state provider, you can implement this class. - """ - - @abstractmethod - def get_last_block_number(self) -> int: - """ - Returns the last block number. - """ - pass - - @abstractmethod - def get_block_timestamp(self, block_number: int) -> int: - """ - Returns the timestamp of the block with the given block number. - """ - pass - - @abstractmethod - def get_transactions_to_address( - self, address, block_number: int - ) -> List[Dict[str, Any]]: - """ - Returns all transactions to the given address in the given block number. - """ - pass - - class FunctionCallCrawlerState(ABC): """ Abstract class for function call crawler state. @@ -128,47 +100,6 @@ class PickleFileState(FunctionCallCrawlerState): self.cache_size = 0 -class Web3StateProvider(EthereumStateProvider): - """ - Implementation of EthereumStateProvider with web3. - """ - - def __init__(self, w3: Web3): - self.w3 = w3 - - self.blocks_cache = {} - - def get_transaction_reciept(self, transaction_hash: str) -> Dict[str, Any]: - return self.w3.eth.get_transaction_receipt(transaction_hash) - - def get_last_block_number(self) -> int: - return self.w3.eth.block_number - - def _get_block(self, block_number: int) -> Dict[str, Any]: - if block_number in self.blocks_cache: - return self.blocks_cache[block_number] - block = self.w3.eth.getBlock(block_number, full_transactions=True) - - # clear cache if it grows too large - if len(self.blocks_cache) > 50: - self.blocks_cache = {} - - self.blocks_cache[block_number] = block - return block - - def get_block_timestamp(self, block_number: int) -> int: - block = self._get_block(block_number) - return block["timestamp"] - - def get_transactions_to_address( - self, address: ChecksumAddress, block_number: int - ) -> List[Dict[str, Any]]: - block = self._get_block(block_number) - - all_transactions = block["transactions"] - return [tx for tx in all_transactions if tx["to"] == address] - - # b'\x8d\xa5\xcb[' # Need to utfy because args contains bytes # For now casting to hex, because that byte at top is function signature diff --git a/moonworm/crawler/log_scanner.py b/moonworm/crawler/log_scanner.py index 6c1f4b8..fdd19e1 100644 --- a/moonworm/crawler/log_scanner.py +++ b/moonworm/crawler/log_scanner.py @@ -1,8 +1,9 @@ # Used example scanner from web3 documentation : https://web3py.readthedocs.io/en/stable/examples.html#eth-getlogs-limitations import datetime +import json import logging import time -from typing import Callable, Iterable, List, Optional, Tuple +from typing import Any, Callable, Iterable, List, Optional, Tuple from eth_abi.codec import ABICodec from eth_typing.evm import ChecksumAddress @@ -14,6 +15,7 @@ from web3.exceptions import BlockNotFound from web3.types import ABIEvent, FilterParams from .state import EventScannerState +from .function_call_crawler import utfy_dict logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -67,13 +69,20 @@ def _fetch_events_chunk( from_block: int, to_block: int, addresses: Optional[List[ChecksumAddress]] = None, -) -> Iterable: + on_decode_error: Optional[Callable[[Exception], None]] = None, +) -> List[Any]: """Get events using eth_getLogs API. - This method is detached from any contract instance. + Event structure: + { + "event": Event name, + "args": dictionary of event arguments, + "address": contract address, + "blockNumber": block number, + "transactionHash": transaction hash, + "logIndex": log index + } - This is a stateless method, as opposed to createFilter. - It can be safely called against nodes which do not provide `eth_newFilter` API, like Infura. """ if from_block is None: @@ -86,14 +95,7 @@ def _fetch_events_chunk( # More information here https://eth-abi.readthedocs.io/en/latest/index.html codec: ABICodec = web3.codec - # Here we need to poke a bit into Web3 internals, as this - # functionality is not exposed by default. - # Construct JSON-RPC raw filter presentation based on human readable Python descriptions - # Namely, convert event names to their keccak signatures - # More information here: - # https://github.com/ethereum/web3.py/blob/e176ce0793dafdd0573acc8d4b76425b6eb604ca/web3/_utils/filters.py#L71 - # TODO(yhtiyar): Add argument filters and contract address filters - data_filter_set, event_filter_params = construct_event_filter_params( + _, event_filter_params = construct_event_filter_params( event_abi, codec, fromBlock=from_block, @@ -107,15 +109,20 @@ def _fetch_events_chunk( # Convert raw binary data to Python proxy objects as described by ABI all_events = [] for log in logs: - # Convert raw JSON-RPC log result to human readable event by using ABI data - # More information how processLog works here - # https://github.com/ethereum/web3.py/blob/fbaf1ad11b0c7fac09ba34baff2c256cffe0a148/web3/_utils/events.py#L200 try: - evt = get_event_data(codec, event_abi, log) - # Note: This was originally yield, - # but deferring the timeout exception caused the throttle logic not to work - all_events.append(evt) - except: + raw_event = get_event_data(codec, event_abi, log) + event = { + "event": raw_event["event"], + "args": json.loads(Web3.toJSON(utfy_dict(dict(raw_event["args"])))), + "address": raw_event["address"], + "blockNumber": raw_event["blockNumber"], + "transactionHash": raw_event["transactionHash"].hex(), + "logIndex": raw_event["logIndex"], + } + all_events.append(event) + except Exception as e: + if on_decode_error: + on_decode_error(e) continue return all_events diff --git a/moonworm/crawler/moonstream_ethereum_state_provider.py b/moonworm/crawler/moonstream_ethereum_state_provider.py new file mode 100644 index 0000000..24b9319 --- /dev/null +++ b/moonworm/crawler/moonstream_ethereum_state_provider.py @@ -0,0 +1,169 @@ +import logging +from typing import Any, Dict, List, Optional, Union +from eth_typing.evm import ChecksumAddress +from hexbytes.main import HexBytes +from moonstreamdb.db import yield_db_session_ctx +from moonstreamdb.models import ( + EthereumLabel, + EthereumTransaction, + PolygonLabel, + PolygonTransaction, +) +from sqlalchemy.orm import Session +from sqlalchemy.sql.base import NO_ARG +from web3 import Web3 + +from .networks import Network, MODELS +from .ethereum_state_provider import EthereumStateProvider + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# TODO(yhtiyar) When getting block from db, filter it by `to` address, it will be faster +# also get blocks in bunch +class MoonstreamEthereumStateProvider(EthereumStateProvider): + """ + Implementation of EthereumStateProvider with moonstream. + """ + + def __init__( + self, w3: Web3, network: Network, db_session: Optional[Session] = None + ): + self.w3 = w3 + self.db_session = db_session + + self.blocks_model = MODELS[network]["blocks"] + self.transactions_model = MODELS[network]["transactions"] + self.labels_model = MODELS[network]["labels"] + self.network = network + self.blocks_cache = {} + + def set_db_session(self, db_session: Session): + self.db_session = db_session + + def clear_db_session(self): + self.db_session = None + + def get_transaction_reciept(self, transaction_hash: str) -> Dict[str, Any]: + return self.w3.eth.get_transaction_receipt(transaction_hash) + + def get_last_block_number(self) -> int: + last_block = ( + self.db_session.query(self.blocks_model) + .order_by(self.blocks_model.block_number.desc()) + .limit(1) + .one_or_none() + ) + if last_block is None: + raise Exception( + f"No blocks in database, for network: {self.network.value} " + ) + return last_block.block_number + + @staticmethod + def _transform_to_w3_tx( + tx_raw: Union[EthereumTransaction, PolygonTransaction], + ) -> Dict[str, Any]: + tx = { + "blockNumber": tx_raw.block_number, + "from": tx_raw.from_address, + "gas": tx_raw.gas, + "gasPrice": tx_raw.gas_price, + "hash": HexBytes(tx_raw.hash), + "input": tx_raw.input, + "maxFeePerGas": tx_raw.max_fee_per_gas, + "maxPriorityFeePerGas": tx_raw.max_priority_fee_per_gas, + "nonce": tx_raw.nonce, + "to": tx_raw.to_address, + "transactionIndex": tx_raw.transaction_index, + "value": tx_raw.value, + } + return tx + + def _get_block_from_db( + self, block_number: int, batch_load_count: int = 100 + ) -> Optional[Dict[str, Any]]: + if self.db_session is None: + return None + + raw_blocks = ( + self.db_session.query(self.blocks_model) + .filter(self.blocks_model.block_number >= block_number) + .order_by(self.blocks_model.block_number.asc()) + .limit(batch_load_count) + ) + blocks = {raw_block.block_number: raw_block for raw_block in raw_blocks} + + if blocks.get(block_number) is None: + return None + # Assuming that all tx's from a block are written to db in the same db transaction + raw_block_transactions = ( + self.db_session.query(self.transactions_model) + .filter( + self.transactions_model.block_number.in_( + [block_number for block_number in blocks] + ) + ) + .order_by(self.transactions_model.transaction_index.asc()) + .all() + ) + + block_transactions = {} + + for raw_tx in raw_block_transactions: + if block_transactions.get(raw_tx.block_number) is None: + block_transactions[raw_tx.block_number] = [] + block_transactions[raw_tx.block_number].append(raw_tx) + + if block_transactions.get(block_number) is None: + return None + + if len(self.blocks_cache) > 500: + self.blocks_cache = {} + + for block, txs in block_transactions.items(): + self.blocks_cache[block] = { + "timestamp": blocks[block].timestamp, + "transactions": [self._transform_to_w3_tx(tx) for tx in txs], + } + + return self.blocks_cache[block_number] + + def _get_block(self, block_number: int) -> Dict[str, Any]: + log_prefix = f"MoonstreamEthereumStateProvider._get_block: block_number={block_number},network={self.network.value}" + logger.debug(log_prefix) + if block_number in self.blocks_cache: + logger.debug(f"{log_prefix} - found in cache") + return self.blocks_cache[block_number] + + block = self._get_block_from_db(block_number) + if block is None: + logger.debug(f"{log_prefix} - not found in db or cache, fetching from web3") + block = self.w3.eth.getBlock(block_number, full_transactions=True) + else: + logger.debug(f"{log_prefix} - found in db") + + # clear cache if it grows too large + if len(self.blocks_cache) > 500: + self.blocks_cache = {} + + self.blocks_cache[block_number] = block + return block + + def get_block_timestamp(self, block_number: int) -> int: + logger.debug( + f"MoonstreamEthereumStateProvider.get_block_timestamp: block_number={block_number},network={self.network.value}" + ) + block = self._get_block(block_number) + return block["timestamp"] + + def get_transactions_to_address( + self, address: ChecksumAddress, block_number: int + ) -> List[Dict[str, Any]]: + logger.debug( + f"MoonstreamEthereumStateProvider.get_transactions_to_address: address={address},block_number={block_number},network={self.network.value}" + ) + block = self._get_block(block_number) + + all_transactions = block["transactions"] + return [tx for tx in all_transactions if tx["to"] == address] diff --git a/moonworm/crawler/networks.py b/moonworm/crawler/networks.py new file mode 100644 index 0000000..7c81e04 --- /dev/null +++ b/moonworm/crawler/networks.py @@ -0,0 +1,30 @@ +from enum import Enum +from typing import Dict +from moonstreamdb.models import ( + Base, + EthereumBlock, + EthereumLabel, + EthereumTransaction, + PolygonBlock, + PolygonLabel, + PolygonTransaction, +) + + +class Network(Enum): + ethereum = "ethereum" + polygon = "polygon" + + +MODELS: Dict[Network, Dict[str, Base]] = { + Network.ethereum: { + "blocks": EthereumBlock, + "labels": EthereumLabel, + "transactions": EthereumTransaction, + }, + Network.polygon: { + "blocks": PolygonBlock, + "labels": PolygonLabel, + "transactions": PolygonTransaction, + }, +} diff --git a/moonworm/cu_current_cuts.py b/moonworm/cu_current_cuts.py index cad9ec8..164c8e0 100644 --- a/moonworm/cu_current_cuts.py +++ b/moonworm/cu_current_cuts.py @@ -58,7 +58,7 @@ def get_all_diamond_cuts(session: Session): labels = ( session.query(PolygonLabel.label_data) .filter(PolygonLabel.label == "moonworm") - .filter(PolygonLabel.address == MUMBAI_ADDRESS) + .filter(PolygonLabel.address == ADDRESS) .filter(PolygonLabel.label_data["name"].astext == "diamondCut") .filter(PolygonLabel.label_data["status"].astext == "1") .order_by(PolygonLabel.block_number.asc()) diff --git a/moonworm/cu_watch.py b/moonworm/cu_watch.py index 9783129..721e4f0 100644 --- a/moonworm/cu_watch.py +++ b/moonworm/cu_watch.py @@ -14,12 +14,18 @@ from sqlalchemy.orm import Query, Session from tqdm import tqdm from web3 import Web3 +from moonworm.crawler.moonstream_ethereum_state_provider import ( + MoonstreamEthereumStateProvider, +) +from moonworm.crawler.networks import Network + from .contracts import CU, ERC721 from .crawler.function_call_crawler import ( ContractFunctionCall, FunctionCallCrawler, FunctionCallCrawlerState, Web3StateProvider, + utfy_dict, ) from .crawler.log_scanner import _fetch_events_chunk @@ -50,6 +56,7 @@ def _get_last_crawled_block( def _add_function_call_labels( session: Session, function_calls: List[ContractFunctionCall], + address: ChecksumAddress, ) -> None: """ Adds a label to a function call. @@ -60,6 +67,7 @@ def _add_function_call_labels( .filter( PolygonLabel.label == "moonworm", PolygonLabel.log_index == None, + PolygonLabel.address == address, PolygonLabel.transaction_hash.in_( [call.transaction_hash for call in function_calls] ), @@ -73,7 +81,7 @@ def _add_function_call_labels( try: if existing_function_call_labels: logger.info( - f"Deleting {len(existing_function_call_labels)} existing tx labels" + f"Deleting {len(existing_function_call_labels)} existing tx labels:\n{existing_function_call_labels}" ) session.commit() except Exception as e: @@ -110,17 +118,21 @@ def _add_function_call_labels( session.rollback() -def _add_event_labels(session: Session, events: List[Dict[str, Any]]) -> None: +def _add_event_labels( + session: Session, events: List[Dict[str, Any]], address: ChecksumAddress +) -> None: """ Adds events to database. """ transactions = [event["transactionHash"] for event in events] - + for ev in events: + print(ev) existing_event_labels = ( session.query(PolygonLabel) .filter( PolygonLabel.label == "moonworm", + PolygonLabel.address == address, PolygonLabel.transaction_hash.in_(transactions), PolygonLabel.log_index != None, ) @@ -133,7 +145,9 @@ def _add_event_labels(session: Session, events: List[Dict[str, Any]]) -> None: try: if existing_event_labels: - logger.error(f"Deleting {len(existing_event_labels)} existing event labels") + logger.error( + f"Deleting {len(existing_event_labels)} existing event labels:\n{existing_event_labels}" + ) session.commit() except Exception as e: try: @@ -198,6 +212,7 @@ def watch_cu_contract( sleep_time: float = 1, start_block: Optional[int] = None, force_start: bool = False, + use_moonstream_web3_provider: bool = False, ) -> None: """ Watches a contract for events and calls. @@ -206,10 +221,17 @@ def watch_cu_contract( raise ValueError("start_block must be specified if force_start is True") with yield_db_session_ctx() as session: - state = MockState() + function_call_state = MockState() + web3_state = Web3StateProvider(web3) + eth_state_provider = web3_state + if use_moonstream_web3_provider: + eth_state_provider = MoonstreamEthereumStateProvider( + web3, network=Network.polygon, session=session + ) + crawler = FunctionCallCrawler( - state, - Web3StateProvider(web3), + function_call_state, + eth_state_provider, contract_abi, [web3.toChecksumAddress(contract_address)], ) @@ -231,16 +253,14 @@ def watch_cu_contract( ) else: current_block = last_crawled_block - logger.info(f"Starting from last crawled block {start_block}") + logger.info(f"Starting from last crawled block {current_block}") event_abis = [item for item in contract_abi if item["type"] == "event"] while True: session.query(PolygonLabel).limit(1).one() time.sleep(sleep_time) - end_block = min( - web3.eth.blockNumber - num_confirmations, current_block + 100 - ) + end_block = min(web3.eth.blockNumber - num_confirmations, current_block + 5) if end_block < current_block: sleep_time *= 2 continue @@ -249,12 +269,15 @@ def watch_cu_contract( logger.info("Getting txs") crawler.crawl(current_block, end_block) - if state.state: - _add_function_call_labels(session, state.state) - logger.info(f"Got {len(state.state)} transaction calls:") - state.flush() + if function_call_state.state: + _add_function_call_labels( + session, function_call_state.state, contract_address + ) + logger.info(f"Got {len(function_call_state.state)} transaction calls:") + function_call_state.flush() - logger.info("Getting") + logger.info("Getting events") + all_events = [] for event_abi in event_abis: raw_events = _fetch_events_chunk( web3, @@ -263,11 +286,14 @@ def watch_cu_contract( end_block, [contract_address], ) - all_events = [] + for raw_event in raw_events: + event = { "event": raw_event["event"], - "args": json.loads(Web3.toJSON(raw_event["args"])), + "args": json.loads( + Web3.toJSON(utfy_dict(dict(raw_event["args"]))) + ), "address": raw_event["address"], "blockNumber": raw_event["blockNumber"], "transactionHash": raw_event["transactionHash"].hex(), @@ -278,7 +304,8 @@ def watch_cu_contract( } all_events.append(event) - if all_events: - _add_event_labels(session, all_events) + if all_events: + print(f"Got {len(all_events)} events:") + _add_event_labels(session, all_events, contract_address) logger.info(f"Current block {end_block + 1}") current_block = end_block + 1 diff --git a/moonworm/fixture/abis/CUContract.json b/moonworm/fixture/abis/CUContract.json index c11274d..7643cd0 100644 --- a/moonworm/fixture/abis/CUContract.json +++ b/moonworm/fixture/abis/CUContract.json @@ -1134,5 +1134,624 @@ "outputs": [], "stateMutability": "nonpayable", "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_classId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "_statId", + "type": "uint256" + } + ], + "name": "getBaseStats", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_classId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "_partSlotId", + "type": "uint256" + } + ], + "name": "getBodyPartBuckets", + "outputs": [ + { + "internalType": "uint256[]", + "name": "", + "type": "uint256[]" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_classId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "_partSlotId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "_localPartId", + "type": "uint256" + } + ], + "name": "getBodyPartGlobalIdFromLocalId", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_globalPartId", + "type": "uint256" + } + ], + "name": "getBodyPartInheritedGene", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_globalPartId", + "type": "uint256" + } + ], + "name": "getBodyPartIsMythic", + "outputs": [ + { + "internalType": "bool", + "name": "", + "type": "bool" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_globalPartId", + "type": "uint256" + } + ], + "name": "getBodyPartLocalIdFromGlobalId", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_globalPartId", + "type": "uint256" + } + ], + "name": "getBodyPartWeight", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_geneId", + "type": "uint256" + } + ], + "name": "getGeneApplicationById", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_geneId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "_geneBonusSlot", + "type": "uint256" + } + ], + "name": "getGeneBonusStatByGeneId", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_geneId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "_geneBonusSlot", + "type": "uint256" + } + ], + "name": "getGeneBonusValueByGeneId", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_classId", + "type": "uint256" + } + ], + "name": "getGeneBucketSumWeights", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_classId", + "type": "uint256" + } + ], + "name": "getGeneBuckets", + "outputs": [ + { + "internalType": "uint256[]", + "name": "", + "type": "uint256[]" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_geneId", + "type": "uint256" + } + ], + "name": "getGeneTierById", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_geneId", + "type": "uint256" + } + ], + "name": "getGeneTierUpgradeById", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_geneId", + "type": "uint256" + } + ], + "name": "getGeneWeightById", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_dna", + "type": "uint256" + } + ], + "name": "getAccuracy", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_dna", + "type": "uint256" + } + ], + "name": "getAttack", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_dna", + "type": "uint256" + } + ], + "name": "getAttackSpeed", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_dna", + "type": "uint256" + } + ], + "name": "getDefense", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_dna", + "type": "uint256" + } + ], + "name": "getMagic", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_dna", + "type": "uint256" + } + ], + "name": "getMovementSpeed", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_dna", + "type": "uint256" + } + ], + "name": "getResistance", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_dna", + "type": "uint256" + } + ], + "name": "getStats", + "outputs": [ + { + "internalType": "uint256", + "name": "attack", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "accuracy", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "movementSpeed", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "attackSpeed", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "defense", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "vitality", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "resistance", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "magic", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_dna", + "type": "uint256" + } + ], + "name": "getUnicornBodyParts", + "outputs": [ + { + "internalType": "uint256", + "name": "bodyPartId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "facePartId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "hornPartId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "hoovesPartId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "manePartId", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "tailPartId", + "type": "uint256" + }, + { + "internalType": "uint8", + "name": "mythicCount", + "type": "uint8" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_tokenId", + "type": "uint256" + } + ], + "name": "getUnicornMetadata", + "outputs": [ + { + "internalType": "bool", + "name": "origin", + "type": "bool" + }, + { + "internalType": "bool", + "name": "gameLocked", + "type": "bool" + }, + { + "internalType": "bool", + "name": "limitedEdition", + "type": "bool" + }, + { + "internalType": "uint256", + "name": "lifecycleStage", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "breedingPoints", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "unicornClass", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "hatchBirthday", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_dna", + "type": "uint256" + } + ], + "name": "getVitality", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" } ] \ No newline at end of file diff --git a/moonworm/watch.py b/moonworm/watch.py index f2982fe..a96c507 100644 --- a/moonworm/watch.py +++ b/moonworm/watch.py @@ -7,6 +7,8 @@ from eth_typing.evm import ChecksumAddress from tqdm import tqdm from web3 import Web3 +from moonworm.crawler.ethereum_state_provider import EthereumStateProvider + from .contracts import CU, ERC721 from .crawler.function_call_crawler import ( ContractFunctionCall, @@ -40,8 +42,10 @@ class MockState(FunctionCallCrawlerState): self.state = [] +# TODO(yhtiyar), use state_provider.get_last_block def watch_contract( web3: Web3, + state_provider: EthereumStateProvider, contract_address: ChecksumAddress, contract_abi: List[Dict[str, Any]], num_confirmations: int = 10, @@ -54,7 +58,7 @@ def watch_contract( state = MockState() crawler = FunctionCallCrawler( state, - Web3StateProvider(web3), + state_provider, contract_abi, [web3.toChecksumAddress(contract_address)], ) @@ -70,14 +74,14 @@ def watch_contract( progress_bar.set_description(f"Current block {current_block}") while True: time.sleep(sleep_time) - end_block = web3.eth.blockNumber - num_confirmations + end_block = min(web3.eth.blockNumber - num_confirmations, current_block + 100) if end_block < current_block: sleep_time *= 2 continue sleep_time /= 2 - crawler.crawl(current_block, end_block) + # crawler.crawl(current_block, end_block) if state.state: print("Got transaction calls:") for call in state.state: