Cleaned up NFT crawler code a bit

Hooked it up to comamnd line

crawlers/mooncrawl/.gitignore

@@ -0,0 +1,2 @@
+.venv/
+.mooncrawl/

crawlers/mooncrawl/mooncrawl/eth_nft_explorer.py

@@ -1,14 +1,14 @@
 from dataclasses import dataclass, asdict
-from collections import defaultdict
+from typing import cast, List, Optional
-from typing import Dict, List, Optional
+from hexbytes.main import HexBytes
+
+from eth_typing.encoding import HexStr
+from tqdm import tqdm
 from web3 import Web3
-import web3
+from web3.types import FilterParams, LogReceipt
-from web3.types import FilterParams
 from web3._utils.events import get_event_data
 
 
-w3 = Web3(Web3.HTTPProvider("http://127.0.0.1:18375"))
-
 # First abi is for old NFT's like crypto kitties
 # The erc721 standart requieres that Transfer event is indexed for all arguments
 # That is how we get distinguished from erc20 transfer events
@@ -89,7 +89,7 @@ class NFT_contract:
     total_supply: str
 
 
-def get_erc721_contract_info(address: str) -> NFT_contract:
+def get_erc721_contract_info(w3: Web3, address: str) -> NFT_contract:
     contract = w3.eth.contract(
         address=w3.toChecksumAddress(address), abi=erc721_functions_abi
     )
@@ -101,7 +101,10 @@ def get_erc721_contract_info(address: str) -> NFT_contract:
     )
 
 
-transfer_event_signature = w3.sha3(text="Transfer(address,address,uint256)").hex()
+# SHA3 hash of the string "Transfer(address,address,uint256)"
+TRANSFER_EVENT_SIGNATURE = HexBytes(
+    "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
+)
 
 
 @dataclass
@@ -110,7 +113,7 @@ class NFTTransferRaw:
     transfer_from: str
     transfer_to: str
     tokenId: int
-    transfer_tx: bytes
+    transfer_tx: HexBytes
 
 
 @dataclass
@@ -124,14 +127,14 @@ class NFTTransfer:
     is_mint: bool = False
 
 
-def get_value_by_tx(tx_hash):
+def get_value_by_tx(w3: Web3, tx_hash: HexBytes):
     print(f"Trying to get tx: {tx_hash.hex()}")
     tx = w3.eth.get_transaction(tx_hash)
     print("got it")
     return tx["value"]
 
 
-def decode_nft_transfer_data(log) -> Optional[NFTTransferRaw]:
+def decode_nft_transfer_data(w3: Web3, log: LogReceipt) -> Optional[NFTTransferRaw]:
     for abi in erc721_transfer_event_abis:
         try:
             transfer_data = get_event_data(w3.codec, abi, log)
@@ -149,47 +152,34 @@ def decode_nft_transfer_data(log) -> Optional[NFTTransferRaw]:
 
 
 def get_nft_transfers(
-    block_number_from: int, contract_address: Optional[str] = None
+    w3: Web3,
+    from_block: Optional[int] = None,
+    to_block: Optional[int] = None,
+    contract_address: Optional[str] = None,
 ) -> List[NFTTransfer]:
-    filter_params = FilterParams(
+    filter_params = FilterParams(topics=[cast(HexStr, TRANSFER_EVENT_SIGNATURE.hex())])
-        fromBlock=block_number_from, topics=[transfer_event_signature]
+
-    )
+    if from_block is not None:
+        filter_params["fromBlock"] = from_block
+
+    if to_block is not None:
+        filter_params["toBlock"] = to_block
 
     if contract_address is not None:
         filter_params["address"] = w3.toChecksumAddress(contract_address)
 
     logs = w3.eth.get_logs(filter_params)
     nft_transfers: List[NFTTransfer] = []
-    tx_value: Dict[bytes, List[NFTTransferRaw]] = defaultdict(list)
+    for log in tqdm(logs):
-    for log in logs:
+        nft_transfer = decode_nft_transfer_data(w3, log)
-        nft_transfer = decode_nft_transfer_data(log)
         if nft_transfer is not None:
-            tx_value[nft_transfer.transfer_tx].append(nft_transfer)
-
-    for tx_hash, transfers in tx_value.items():
-        # value = get_value_by_tx(tx_hash)
-        value = 0
-        for transfer in transfers:
             kwargs = {
-                **asdict(transfer),
+                **asdict(nft_transfer),
-                "transfer_tx": transfer.transfer_tx.hex(),
+                "transfer_tx": nft_transfer.transfer_tx.hex(),
-                "is_mint": transfer.transfer_from
+                "is_mint": nft_transfer.transfer_from
                 == "0x0000000000000000000000000000000000000000",
-                "value": value,
             }
-            parsed_transfer = NFTTransfer(**kwargs)
 
+            parsed_transfer = NFTTransfer(**kwargs)  # type: ignore
             nft_transfers.append(parsed_transfer)
     return nft_transfers
-
-
-cryptoKittiesAddress = "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d"
-transfesrs = get_nft_transfers(
-    w3.eth.block_number - 120,
-)
-
-print(transfesrs)
-print(f"Total nft transfers: {len(transfesrs)}")
-minted = list(filter(lambda transfer: transfer.is_mint == True, transfesrs))
-# print(minted)
-print(f"Minted count: {len(minted)}")

crawlers/mooncrawl/mooncrawl/ethcrawler.py

@@ -8,11 +8,13 @@ import json
 import os
 import sys
 import time
-from typing import Iterator, List
+from typing import cast, Iterator, List
 
 import dateutil.parser
+from web3 import Web3
 
 from .ethereum import (
+    connect,
     crawl_blocks_executor,
     crawl_blocks,
     check_missing_blocks,
@@ -21,8 +23,9 @@ from .ethereum import (
     DateRange,
     trending,
 )
+from .eth_nft_explorer import get_nft_transfers
 from .publish import publish_json
-from .settings import MOONSTREAM_CRAWL_WORKERS
+from .settings import MOONSTREAM_CRAWL_WORKERS, MOONSTREAM_IPC_PATH
 from .version import MOONCRAWL_VERSION
 
 
@@ -31,6 +34,22 @@ class ProcessingOrder(Enum):
     ASCENDING = 1
 
 
+def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3:
+    """
+    Returns a web3 client either by parsing "--web3" argument on the given arguments or by looking up
+    the MOONSTREAM_IPC_PATH environment variable.
+    """
+    web3_connection_string = MOONSTREAM_IPC_PATH
+    args_web3 = vars(args).get("web3")
+    if args_web3 is not None:
+        web3_connection_string = cast(str, args_web3)
+    if web3_connection_string is None:
+        raise ValueError(
+            "Could not find Web3 connection information in arguments or in MOONSTREAM_IPC_PATH environment variable"
+        )
+    return connect(web3_connection_string)
+
+
 def yield_blocks_numbers_lists(
     blocks_range_str: str,
     order: ProcessingOrder = ProcessingOrder.DESCENDING,
@@ -200,6 +219,16 @@ def ethcrawler_trending_handler(args: argparse.Namespace) -> None:
         json.dump(results, ofp)
 
 
+def ethcrawler_nft_handler(args: argparse.Namespace) -> None:
+    web3_client = web3_client_from_cli_or_env(args)
+    transfers = get_nft_transfers(web3_client, args.start, args.end, args.address)
+    for transfer in transfers:
+        print(transfer)
+
+    print("Total transfers:", len(transfers))
+    print("Mints:", len([transfer for transfer in transfers if transfer.is_mint]))
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description="Moonstream crawlers CLI")
     parser.set_defaults(func=lambda _: parser.print_help())
@@ -389,6 +418,38 @@ def main() -> None:
     )
     parser_ethcrawler_trending.set_defaults(func=ethcrawler_trending_handler)
 
+    parser_ethcrawler_nft = subcommands.add_parser(
+        "nft", description="Collect information about NFTs from Ethereum blockchains"
+    )
+    parser_ethcrawler_nft.add_argument(
+        "-s",
+        "--start",
+        type=int,
+        default=None,
+        help="Starting block number (inclusive if block available)",
+    )
+    parser_ethcrawler_nft.add_argument(
+        "-e",
+        "--end",
+        type=int,
+        default=None,
+        help="Ending block number (inclusive if block available)",
+    )
+    parser_ethcrawler_nft.add_argument(
+        "-a",
+        "--address",
+        type=str,
+        default=None,
+        help="(Optional) NFT contract address that you want to limit the crawl to, e.g. 0x06012c8cf97BEaD5deAe237070F9587f8E7A266d for CryptoKitties.",
+    )
+    parser_ethcrawler_nft.add_argument(
+        "--web3",
+        type=str,
+        default=None,
+        help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_IPC_PATH environment variable.",
+    )
+    parser_ethcrawler_nft.set_defaults(func=ethcrawler_nft_handler)
+
     args = parser.parse_args()
     args.func(args)
 

crawlers/mooncrawl/mypy.ini

@@ -8,3 +8,6 @@ ignore_missing_imports = True
 
 [mypy-pyevmasm.*]
 ignore_missing_imports = True
+
+[mypy-tqdm.*]
+ignore_missing_imports = True