adding of ethereum moonstream provider

moonworm/cli.py

@@ -6,6 +6,7 @@ from shutil import copyfile
 
 from web3.main import Web3
 from web3.middleware import geth_poa_middleware
+from moonworm.crawler.ethereum_state_provider import Web3StateProvider
 
 from moonworm.watch import watch_contract
 
@@ -14,6 +15,7 @@ from .generator import (
     generate_contract_cli_content,
     generate_contract_interface_content,
 )
+from .crawler.networks import Network
 
 
 def write_file(content: str, path: str):
@@ -92,9 +94,41 @@ def handle_watch(args: argparse.Namespace) -> None:
     web3 = Web3(Web3.HTTPProvider(args.web3))
     if args.poa:
         web3.middleware_onion.inject(geth_poa_middleware, layer=0)
-    watch_contract(
+    if args.db:
-        web3, web3.toChecksumAddress(args.contract), contract_abi, args.confirmations
+        if args.network is None:
-    )
+            raise ValueError("Please specify --network")
+        network = Network.__members__[args.network]
+        from .crawler.moonstream_ethereum_state_provider import (
+            MoonstreamEthereumStateProvider,
+        )
+        from moonstreamdb.db import yield_db_session_ctx
+
+        state_provider = MoonstreamEthereumStateProvider(web3, network)
+
+        with yield_db_session_ctx() as db_session:
+            try:
+                state_provider.set_db_session(db_session)
+                watch_contract(
+                    web3=web3,
+                    state_provider=state_provider,
+                    contract_address=web3.toChecksumAddress(args.contract),
+                    contract_abi=contract_abi,
+                    num_confirmations=args.confirmations,
+                    start_block=args.start,
+                )
+            finally:
+                state_provider.clear_db_session()
+
+    else:
+        state_provider = Web3StateProvider(web3)
+        watch_contract(
+            web3=web3,
+            state_provider=state_provider,
+            contract_address=web3.toChecksumAddress(args.contract),
+            contract_abi=contract_abi,
+            num_confirmations=args.confirmations,
+            start_block=args.start,
+        )
 
 
 def handle_watch_cu(args: argparse.Namespace) -> None:
@@ -152,6 +186,27 @@ def generate_argument_parser() -> argparse.ArgumentParser:
         help="Web3 provider",
     )
 
+    watch_parser.add_argument(
+        "--db",
+        action="store_true",
+        help="Use Moonstream database specified by 'MOONSTREAM_DB_URI' to get blocks/transactions. If set, need also provide --network",
+    )
+
+    watch_parser.add_argument(
+        "--network",
+        choices=Network.__members__,
+        default=None,
+        help="Network name that represents models from db. If --db is set, required",
+    )
+
+    watch_parser.add_argument(
+        "--start",
+        "-s",
+        type=int,
+        default=None,
+        help="Block number to start watching from",
+    )
+
     watch_parser.add_argument(
         "--poa",
         action="store_true",

moonworm/crawler/ethereum_state_provider.py

@@ -0,0 +1,82 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Union
+import logging
+from eth_typing.evm import ChecksumAddress
+
+
+from web3 import Web3
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class EthereumStateProvider(ABC):
+    """
+    Abstract class for Ethereum state provider.
+    If you want to use a different state provider, you can implement this class.
+    """
+
+    @abstractmethod
+    def get_last_block_number(self) -> int:
+        """
+        Returns the last block number.
+        """
+        pass
+
+    @abstractmethod
+    def get_block_timestamp(self, block_number: int) -> int:
+        """
+        Returns the timestamp of the block with the given block number.
+        """
+        pass
+
+    @abstractmethod
+    def get_transactions_to_address(
+        self, address, block_number: int
+    ) -> List[Dict[str, Any]]:
+        """
+        Returns all transactions to the given address in the given block number.
+        """
+        pass
+
+
+class Web3StateProvider(EthereumStateProvider):
+    """
+    Implementation of EthereumStateProvider with web3.
+    """
+
+    def __init__(self, w3: Web3):
+        self.w3 = w3
+
+        self.blocks_cache = {}
+
+    def get_transaction_reciept(self, transaction_hash: str) -> Dict[str, Any]:
+        return self.w3.eth.get_transaction_receipt(transaction_hash)
+
+    def get_last_block_number(self) -> int:
+        return self.w3.eth.block_number
+
+    def _get_block(self, block_number: int) -> Dict[str, Any]:
+        if block_number in self.blocks_cache:
+            return self.blocks_cache[block_number]
+        block = self.w3.eth.getBlock(block_number, full_transactions=True)
+
+        # clear cache if it grows too large
+        if len(self.blocks_cache) > 50:
+            self.blocks_cache = {}
+
+        self.blocks_cache[block_number] = block
+        return block
+
+    def get_block_timestamp(self, block_number: int) -> int:
+        block = self._get_block(block_number)
+        return block["timestamp"]
+
+    def get_transactions_to_address(
+        self, address: ChecksumAddress, block_number: int
+    ) -> List[Dict[str, Any]]:
+        block = self._get_block(block_number)
+
+        all_transactions = block["transactions"]
+        return [tx for tx in all_transactions if tx["to"] == address]

moonworm/crawler/function_call_crawler.py

@@ -11,6 +11,8 @@ from web3.types import ABI
 
 from moonworm.contracts import ERC1155
 
+from .ethereum_state_provider import EthereumStateProvider, Web3StateProvider
+
 
 @dataclass
 class ContractFunctionCall:
@@ -25,36 +27,6 @@ class ContractFunctionCall:
     status: int
 
 
-class EthereumStateProvider(ABC):
-    """
-    Abstract class for Ethereum state provider.
-    If you want to use a different state provider, you can implement this class.
-    """
-
-    @abstractmethod
-    def get_last_block_number(self) -> int:
-        """
-        Returns the last block number.
-        """
-        pass
-
-    @abstractmethod
-    def get_block_timestamp(self, block_number: int) -> int:
-        """
-        Returns the timestamp of the block with the given block number.
-        """
-        pass
-
-    @abstractmethod
-    def get_transactions_to_address(
-        self, address, block_number: int
-    ) -> List[Dict[str, Any]]:
-        """
-        Returns all transactions to the given address in the given block number.
-        """
-        pass
-
-
 class FunctionCallCrawlerState(ABC):
     """
     Abstract class for function call crawler state.
@@ -128,47 +100,6 @@ class PickleFileState(FunctionCallCrawlerState):
         self.cache_size = 0
 
 
-class Web3StateProvider(EthereumStateProvider):
-    """
-    Implementation of EthereumStateProvider with web3.
-    """
-
-    def __init__(self, w3: Web3):
-        self.w3 = w3
-
-        self.blocks_cache = {}
-
-    def get_transaction_reciept(self, transaction_hash: str) -> Dict[str, Any]:
-        return self.w3.eth.get_transaction_receipt(transaction_hash)
-
-    def get_last_block_number(self) -> int:
-        return self.w3.eth.block_number
-
-    def _get_block(self, block_number: int) -> Dict[str, Any]:
-        if block_number in self.blocks_cache:
-            return self.blocks_cache[block_number]
-        block = self.w3.eth.getBlock(block_number, full_transactions=True)
-
-        # clear cache if it grows too large
-        if len(self.blocks_cache) > 50:
-            self.blocks_cache = {}
-
-        self.blocks_cache[block_number] = block
-        return block
-
-    def get_block_timestamp(self, block_number: int) -> int:
-        block = self._get_block(block_number)
-        return block["timestamp"]
-
-    def get_transactions_to_address(
-        self, address: ChecksumAddress, block_number: int
-    ) -> List[Dict[str, Any]]:
-        block = self._get_block(block_number)
-
-        all_transactions = block["transactions"]
-        return [tx for tx in all_transactions if tx["to"] == address]
-
-
 # b'\x8d\xa5\xcb['
 # Need to utfy because args contains bytes
 # For now casting to hex, because that byte at top is function signature

moonworm/crawler/log_scanner.py

@@ -1,8 +1,9 @@
 # Used example scanner from web3 documentation : https://web3py.readthedocs.io/en/stable/examples.html#eth-getlogs-limitations
 import datetime
+import json
 import logging
 import time
-from typing import Callable, Iterable, List, Optional, Tuple
+from typing import Any, Callable, Iterable, List, Optional, Tuple
 
 from eth_abi.codec import ABICodec
 from eth_typing.evm import ChecksumAddress
@@ -14,6 +15,7 @@ from web3.exceptions import BlockNotFound
 from web3.types import ABIEvent, FilterParams
 
 from .state import EventScannerState
+from .function_call_crawler import utfy_dict
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -67,13 +69,20 @@ def _fetch_events_chunk(
     from_block: int,
     to_block: int,
     addresses: Optional[List[ChecksumAddress]] = None,
-) -> Iterable:
+    on_decode_error: Optional[Callable[[Exception], None]] = None,
+) -> List[Any]:
     """Get events using eth_getLogs API.
 
-    This method is detached from any contract instance.
+    Event structure:
+    {
+        "event": Event name,
+        "args": dictionary of event arguments,
+        "address": contract address,
+        "blockNumber": block number,
+        "transactionHash": transaction hash,
+        "logIndex": log index
+    }
 
-    This is a stateless method, as opposed to createFilter.
-    It can be safely called against nodes which do not provide `eth_newFilter` API, like Infura.
     """
 
     if from_block is None:
@@ -86,14 +95,7 @@ def _fetch_events_chunk(
     # More information here https://eth-abi.readthedocs.io/en/latest/index.html
     codec: ABICodec = web3.codec
 
-    # Here we need to poke a bit into Web3 internals, as this
+    _, event_filter_params = construct_event_filter_params(
-    # functionality is not exposed by default.
-    # Construct JSON-RPC raw filter presentation based on human readable Python descriptions
-    # Namely, convert event names to their keccak signatures
-    # More information here:
-    # https://github.com/ethereum/web3.py/blob/e176ce0793dafdd0573acc8d4b76425b6eb604ca/web3/_utils/filters.py#L71
-    # TODO(yhtiyar): Add argument filters and contract address filters
-    data_filter_set, event_filter_params = construct_event_filter_params(
         event_abi,
         codec,
         fromBlock=from_block,
@@ -107,15 +109,20 @@ def _fetch_events_chunk(
     # Convert raw binary data to Python proxy objects as described by ABI
     all_events = []
     for log in logs:
-        # Convert raw JSON-RPC log result to human readable event by using ABI data
-        # More information how processLog works here
-        # https://github.com/ethereum/web3.py/blob/fbaf1ad11b0c7fac09ba34baff2c256cffe0a148/web3/_utils/events.py#L200
         try:
-            evt = get_event_data(codec, event_abi, log)
+            raw_event = get_event_data(codec, event_abi, log)
-            # Note: This was originally yield,
+            event = {
-            # but deferring the timeout exception caused the throttle logic not to work
+                "event": raw_event["event"],
-            all_events.append(evt)
+                "args": json.loads(Web3.toJSON(utfy_dict(dict(raw_event["args"])))),
-        except:
+                "address": raw_event["address"],
+                "blockNumber": raw_event["blockNumber"],
+                "transactionHash": raw_event["transactionHash"].hex(),
+                "logIndex": raw_event["logIndex"],
+            }
+            all_events.append(event)
+        except Exception as e:
+            if on_decode_error:
+                on_decode_error(e)
             continue
     return all_events
 

moonworm/crawler/moonstream_ethereum_state_provider.py

@@ -0,0 +1,169 @@
+import logging
+from typing import Any, Dict, List, Optional, Union
+from eth_typing.evm import ChecksumAddress
+from hexbytes.main import HexBytes
+from moonstreamdb.db import yield_db_session_ctx
+from moonstreamdb.models import (
+    EthereumLabel,
+    EthereumTransaction,
+    PolygonLabel,
+    PolygonTransaction,
+)
+from sqlalchemy.orm import Session
+from sqlalchemy.sql.base import NO_ARG
+from web3 import Web3
+
+from .networks import Network, MODELS
+from .ethereum_state_provider import EthereumStateProvider
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# TODO(yhtiyar) When getting block from db, filter it by `to` address, it will be faster
+# also get blocks in bunch
+class MoonstreamEthereumStateProvider(EthereumStateProvider):
+    """
+    Implementation of EthereumStateProvider with moonstream.
+    """
+
+    def __init__(
+        self, w3: Web3, network: Network, db_session: Optional[Session] = None
+    ):
+        self.w3 = w3
+        self.db_session = db_session
+
+        self.blocks_model = MODELS[network]["blocks"]
+        self.transactions_model = MODELS[network]["transactions"]
+        self.labels_model = MODELS[network]["labels"]
+        self.network = network
+        self.blocks_cache = {}
+
+    def set_db_session(self, db_session: Session):
+        self.db_session = db_session
+
+    def clear_db_session(self):
+        self.db_session = None
+
+    def get_transaction_reciept(self, transaction_hash: str) -> Dict[str, Any]:
+        return self.w3.eth.get_transaction_receipt(transaction_hash)
+
+    def get_last_block_number(self) -> int:
+        last_block = (
+            self.db_session.query(self.blocks_model)
+            .order_by(self.blocks_model.block_number.desc())
+            .limit(1)
+            .one_or_none()
+        )
+        if last_block is None:
+            raise Exception(
+                f"No blocks in database, for network: {self.network.value} "
+            )
+        return last_block.block_number
+
+    @staticmethod
+    def _transform_to_w3_tx(
+        tx_raw: Union[EthereumTransaction, PolygonTransaction],
+    ) -> Dict[str, Any]:
+        tx = {
+            "blockNumber": tx_raw.block_number,
+            "from": tx_raw.from_address,
+            "gas": tx_raw.gas,
+            "gasPrice": tx_raw.gas_price,
+            "hash": HexBytes(tx_raw.hash),
+            "input": tx_raw.input,
+            "maxFeePerGas": tx_raw.max_fee_per_gas,
+            "maxPriorityFeePerGas": tx_raw.max_priority_fee_per_gas,
+            "nonce": tx_raw.nonce,
+            "to": tx_raw.to_address,
+            "transactionIndex": tx_raw.transaction_index,
+            "value": tx_raw.value,
+        }
+        return tx
+
+    def _get_block_from_db(
+        self, block_number: int, batch_load_count: int = 100
+    ) -> Optional[Dict[str, Any]]:
+        if self.db_session is None:
+            return None
+
+        raw_blocks = (
+            self.db_session.query(self.blocks_model)
+            .filter(self.blocks_model.block_number >= block_number)
+            .order_by(self.blocks_model.block_number.asc())
+            .limit(batch_load_count)
+        )
+        blocks = {raw_block.block_number: raw_block for raw_block in raw_blocks}
+
+        if blocks.get(block_number) is None:
+            return None
+        # Assuming that all tx's from a block are written to db in the same db transaction
+        raw_block_transactions = (
+            self.db_session.query(self.transactions_model)
+            .filter(
+                self.transactions_model.block_number.in_(
+                    [block_number for block_number in blocks]
+                )
+            )
+            .order_by(self.transactions_model.transaction_index.asc())
+            .all()
+        )
+
+        block_transactions = {}
+
+        for raw_tx in raw_block_transactions:
+            if block_transactions.get(raw_tx.block_number) is None:
+                block_transactions[raw_tx.block_number] = []
+            block_transactions[raw_tx.block_number].append(raw_tx)
+
+        if block_transactions.get(block_number) is None:
+            return None
+
+        if len(self.blocks_cache) > 500:
+            self.blocks_cache = {}
+
+        for block, txs in block_transactions.items():
+            self.blocks_cache[block] = {
+                "timestamp": blocks[block].timestamp,
+                "transactions": [self._transform_to_w3_tx(tx) for tx in txs],
+            }
+
+        return self.blocks_cache[block_number]
+
+    def _get_block(self, block_number: int) -> Dict[str, Any]:
+        log_prefix = f"MoonstreamEthereumStateProvider._get_block: block_number={block_number},network={self.network.value}"
+        logger.debug(log_prefix)
+        if block_number in self.blocks_cache:
+            logger.debug(f"{log_prefix} - found in cache")
+            return self.blocks_cache[block_number]
+
+        block = self._get_block_from_db(block_number)
+        if block is None:
+            logger.debug(f"{log_prefix} - not found in db or cache, fetching from web3")
+            block = self.w3.eth.getBlock(block_number, full_transactions=True)
+        else:
+            logger.debug(f"{log_prefix} - found in db")
+
+        # clear cache if it grows too large
+        if len(self.blocks_cache) > 500:
+            self.blocks_cache = {}
+
+        self.blocks_cache[block_number] = block
+        return block
+
+    def get_block_timestamp(self, block_number: int) -> int:
+        logger.debug(
+            f"MoonstreamEthereumStateProvider.get_block_timestamp: block_number={block_number},network={self.network.value}"
+        )
+        block = self._get_block(block_number)
+        return block["timestamp"]
+
+    def get_transactions_to_address(
+        self, address: ChecksumAddress, block_number: int
+    ) -> List[Dict[str, Any]]:
+        logger.debug(
+            f"MoonstreamEthereumStateProvider.get_transactions_to_address: address={address},block_number={block_number},network={self.network.value}"
+        )
+        block = self._get_block(block_number)
+
+        all_transactions = block["transactions"]
+        return [tx for tx in all_transactions if tx["to"] == address]

moonworm/crawler/networks.py

@@ -0,0 +1,30 @@
+from enum import Enum
+from typing import Dict
+from moonstreamdb.models import (
+    Base,
+    EthereumBlock,
+    EthereumLabel,
+    EthereumTransaction,
+    PolygonBlock,
+    PolygonLabel,
+    PolygonTransaction,
+)
+
+
+class Network(Enum):
+    ethereum = "ethereum"
+    polygon = "polygon"
+
+
+MODELS: Dict[Network, Dict[str, Base]] = {
+    Network.ethereum: {
+        "blocks": EthereumBlock,
+        "labels": EthereumLabel,
+        "transactions": EthereumTransaction,
+    },
+    Network.polygon: {
+        "blocks": PolygonBlock,
+        "labels": PolygonLabel,
+        "transactions": PolygonTransaction,
+    },
+}

moonworm/cu_current_cuts.py

@@ -58,7 +58,7 @@ def get_all_diamond_cuts(session: Session):
     labels = (
         session.query(PolygonLabel.label_data)
         .filter(PolygonLabel.label == "moonworm")
-        .filter(PolygonLabel.address == MUMBAI_ADDRESS)
+        .filter(PolygonLabel.address == ADDRESS)
         .filter(PolygonLabel.label_data["name"].astext == "diamondCut")
         .filter(PolygonLabel.label_data["status"].astext == "1")
         .order_by(PolygonLabel.block_number.asc())

moonworm/cu_watch.py

@@ -14,12 +14,18 @@ from sqlalchemy.orm import Query, Session
 from tqdm import tqdm
 from web3 import Web3
 
+from moonworm.crawler.moonstream_ethereum_state_provider import (
+    MoonstreamEthereumStateProvider,
+)
+from moonworm.crawler.networks import Network
+
 from .contracts import CU, ERC721
 from .crawler.function_call_crawler import (
     ContractFunctionCall,
     FunctionCallCrawler,
     FunctionCallCrawlerState,
     Web3StateProvider,
+    utfy_dict,
 )
 from .crawler.log_scanner import _fetch_events_chunk
 
@@ -50,6 +56,7 @@ def _get_last_crawled_block(
 def _add_function_call_labels(
     session: Session,
     function_calls: List[ContractFunctionCall],
+    address: ChecksumAddress,
 ) -> None:
     """
     Adds a label to a function call.
@@ -60,6 +67,7 @@ def _add_function_call_labels(
         .filter(
             PolygonLabel.label == "moonworm",
             PolygonLabel.log_index == None,
+            PolygonLabel.address == address,
             PolygonLabel.transaction_hash.in_(
                 [call.transaction_hash for call in function_calls]
             ),
@@ -73,7 +81,7 @@ def _add_function_call_labels(
     try:
         if existing_function_call_labels:
             logger.info(
-                f"Deleting {len(existing_function_call_labels)} existing tx labels"
+                f"Deleting {len(existing_function_call_labels)} existing tx labels:\n{existing_function_call_labels}"
             )
             session.commit()
     except Exception as e:
@@ -110,17 +118,21 @@ def _add_function_call_labels(
             session.rollback()
 
 
-def _add_event_labels(session: Session, events: List[Dict[str, Any]]) -> None:
+def _add_event_labels(
+    session: Session, events: List[Dict[str, Any]], address: ChecksumAddress
+) -> None:
     """
     Adds events to database.
     """
 
     transactions = [event["transactionHash"] for event in events]
-
+    for ev in events:
+        print(ev)
     existing_event_labels = (
         session.query(PolygonLabel)
         .filter(
             PolygonLabel.label == "moonworm",
+            PolygonLabel.address == address,
             PolygonLabel.transaction_hash.in_(transactions),
             PolygonLabel.log_index != None,
         )
@@ -133,7 +145,9 @@ def _add_event_labels(session: Session, events: List[Dict[str, Any]]) -> None:
 
     try:
         if existing_event_labels:
-            logger.error(f"Deleting {len(existing_event_labels)} existing event labels")
+            logger.error(
+                f"Deleting {len(existing_event_labels)} existing event labels:\n{existing_event_labels}"
+            )
             session.commit()
     except Exception as e:
         try:
@@ -198,6 +212,7 @@ def watch_cu_contract(
     sleep_time: float = 1,
     start_block: Optional[int] = None,
     force_start: bool = False,
+    use_moonstream_web3_provider: bool = False,
 ) -> None:
     """
     Watches a contract for events and calls.
@@ -206,10 +221,17 @@ def watch_cu_contract(
         raise ValueError("start_block must be specified if force_start is True")
 
     with yield_db_session_ctx() as session:
-        state = MockState()
+        function_call_state = MockState()
+        web3_state = Web3StateProvider(web3)
+        eth_state_provider = web3_state
+        if use_moonstream_web3_provider:
+            eth_state_provider = MoonstreamEthereumStateProvider(
+                web3, network=Network.polygon, session=session
+            )
+
         crawler = FunctionCallCrawler(
-            state,
+            function_call_state,
-            Web3StateProvider(web3),
+            eth_state_provider,
             contract_abi,
             [web3.toChecksumAddress(contract_address)],
         )
@@ -231,16 +253,14 @@ def watch_cu_contract(
                     )
                 else:
                     current_block = last_crawled_block
-                    logger.info(f"Starting from last crawled block {start_block}")
+                    logger.info(f"Starting from last crawled block {current_block}")
 
         event_abis = [item for item in contract_abi if item["type"] == "event"]
 
         while True:
             session.query(PolygonLabel).limit(1).one()
             time.sleep(sleep_time)
-            end_block = min(
+            end_block = min(web3.eth.blockNumber - num_confirmations, current_block + 5)
-                web3.eth.blockNumber - num_confirmations, current_block + 100
-            )
             if end_block < current_block:
                 sleep_time *= 2
                 continue
@@ -249,12 +269,15 @@ def watch_cu_contract(
 
             logger.info("Getting txs")
             crawler.crawl(current_block, end_block)
-            if state.state:
+            if function_call_state.state:
-                _add_function_call_labels(session, state.state)
+                _add_function_call_labels(
-                logger.info(f"Got  {len(state.state)} transaction calls:")
+                    session, function_call_state.state, contract_address
-                state.flush()
+                )
+                logger.info(f"Got  {len(function_call_state.state)} transaction calls:")
+                function_call_state.flush()
 
-            logger.info("Getting")
+            logger.info("Getting events")
+            all_events = []
             for event_abi in event_abis:
                 raw_events = _fetch_events_chunk(
                     web3,
@@ -263,11 +286,14 @@ def watch_cu_contract(
                     end_block,
                     [contract_address],
                 )
-                all_events = []
+
                 for raw_event in raw_events:
+
                     event = {
                         "event": raw_event["event"],
-                        "args": json.loads(Web3.toJSON(raw_event["args"])),
+                        "args": json.loads(
+                            Web3.toJSON(utfy_dict(dict(raw_event["args"])))
+                        ),
                         "address": raw_event["address"],
                         "blockNumber": raw_event["blockNumber"],
                         "transactionHash": raw_event["transactionHash"].hex(),
@@ -278,7 +304,8 @@ def watch_cu_contract(
                     }
                     all_events.append(event)
 
-                if all_events:
+            if all_events:
-                    _add_event_labels(session, all_events)
+                print(f"Got {len(all_events)} events:")
+                _add_event_labels(session, all_events, contract_address)
             logger.info(f"Current block {end_block + 1}")
             current_block = end_block + 1

moonworm/fixture/abis/CUContract.json

@@ -1134,5 +1134,624 @@
         "outputs": [],
         "stateMutability": "nonpayable",
         "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_classId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "_statId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getBaseStats",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_classId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "_partSlotId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getBodyPartBuckets",
+        "outputs": [
+            {
+                "internalType": "uint256[]",
+                "name": "",
+                "type": "uint256[]"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_classId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "_partSlotId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "_localPartId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getBodyPartGlobalIdFromLocalId",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_globalPartId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getBodyPartInheritedGene",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_globalPartId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getBodyPartIsMythic",
+        "outputs": [
+            {
+                "internalType": "bool",
+                "name": "",
+                "type": "bool"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_globalPartId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getBodyPartLocalIdFromGlobalId",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_globalPartId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getBodyPartWeight",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_geneId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getGeneApplicationById",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_geneId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "_geneBonusSlot",
+                "type": "uint256"
+            }
+        ],
+        "name": "getGeneBonusStatByGeneId",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_geneId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "_geneBonusSlot",
+                "type": "uint256"
+            }
+        ],
+        "name": "getGeneBonusValueByGeneId",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_classId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getGeneBucketSumWeights",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_classId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getGeneBuckets",
+        "outputs": [
+            {
+                "internalType": "uint256[]",
+                "name": "",
+                "type": "uint256[]"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_geneId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getGeneTierById",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_geneId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getGeneTierUpgradeById",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_geneId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getGeneWeightById",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_dna",
+                "type": "uint256"
+            }
+        ],
+        "name": "getAccuracy",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_dna",
+                "type": "uint256"
+            }
+        ],
+        "name": "getAttack",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_dna",
+                "type": "uint256"
+            }
+        ],
+        "name": "getAttackSpeed",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_dna",
+                "type": "uint256"
+            }
+        ],
+        "name": "getDefense",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_dna",
+                "type": "uint256"
+            }
+        ],
+        "name": "getMagic",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_dna",
+                "type": "uint256"
+            }
+        ],
+        "name": "getMovementSpeed",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_dna",
+                "type": "uint256"
+            }
+        ],
+        "name": "getResistance",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_dna",
+                "type": "uint256"
+            }
+        ],
+        "name": "getStats",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "attack",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "accuracy",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "movementSpeed",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "attackSpeed",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "defense",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "vitality",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "resistance",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "magic",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_dna",
+                "type": "uint256"
+            }
+        ],
+        "name": "getUnicornBodyParts",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "bodyPartId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "facePartId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "hornPartId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "hoovesPartId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "manePartId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "tailPartId",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint8",
+                "name": "mythicCount",
+                "type": "uint8"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_tokenId",
+                "type": "uint256"
+            }
+        ],
+        "name": "getUnicornMetadata",
+        "outputs": [
+            {
+                "internalType": "bool",
+                "name": "origin",
+                "type": "bool"
+            },
+            {
+                "internalType": "bool",
+                "name": "gameLocked",
+                "type": "bool"
+            },
+            {
+                "internalType": "bool",
+                "name": "limitedEdition",
+                "type": "bool"
+            },
+            {
+                "internalType": "uint256",
+                "name": "lifecycleStage",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "breedingPoints",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "unicornClass",
+                "type": "uint256"
+            },
+            {
+                "internalType": "uint256",
+                "name": "hatchBirthday",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
+    },
+    {
+        "inputs": [
+            {
+                "internalType": "uint256",
+                "name": "_dna",
+                "type": "uint256"
+            }
+        ],
+        "name": "getVitality",
+        "outputs": [
+            {
+                "internalType": "uint256",
+                "name": "",
+                "type": "uint256"
+            }
+        ],
+        "stateMutability": "view",
+        "type": "function"
     }
 ]

moonworm/watch.py

@@ -7,6 +7,8 @@ from eth_typing.evm import ChecksumAddress
 from tqdm import tqdm
 from web3 import Web3
 
+from moonworm.crawler.ethereum_state_provider import EthereumStateProvider
+
 from .contracts import CU, ERC721
 from .crawler.function_call_crawler import (
     ContractFunctionCall,
@@ -40,8 +42,10 @@ class MockState(FunctionCallCrawlerState):
         self.state = []
 
 
+# TODO(yhtiyar), use state_provider.get_last_block
 def watch_contract(
     web3: Web3,
+    state_provider: EthereumStateProvider,
     contract_address: ChecksumAddress,
     contract_abi: List[Dict[str, Any]],
     num_confirmations: int = 10,
@@ -54,7 +58,7 @@ def watch_contract(
     state = MockState()
     crawler = FunctionCallCrawler(
         state,
-        Web3StateProvider(web3),
+        state_provider,
         contract_abi,
         [web3.toChecksumAddress(contract_address)],
     )
@@ -70,14 +74,14 @@ def watch_contract(
     progress_bar.set_description(f"Current block {current_block}")
     while True:
         time.sleep(sleep_time)
-        end_block = web3.eth.blockNumber - num_confirmations
+        end_block = min(web3.eth.blockNumber - num_confirmations, current_block + 100)
         if end_block < current_block:
             sleep_time *= 2
             continue
 
         sleep_time /= 2
 
-        crawler.crawl(current_block, end_block)
+        # crawler.crawl(current_block, end_block)
         if state.state:
             print("Got transaction calls:")
             for call in state.state: