diff --git a/crawlers/mooncrawl/.gitignore b/crawlers/mooncrawl/.gitignore new file mode 100644 index 00000000..d8fa592b --- /dev/null +++ b/crawlers/mooncrawl/.gitignore @@ -0,0 +1,2 @@ +.venv/ +.mooncrawl/ diff --git a/crawlers/mooncrawl/mooncrawl/eth_nft_explorer.py b/crawlers/mooncrawl/mooncrawl/eth_nft_explorer.py index 98f290f8..5b7c74dd 100644 --- a/crawlers/mooncrawl/mooncrawl/eth_nft_explorer.py +++ b/crawlers/mooncrawl/mooncrawl/eth_nft_explorer.py @@ -1,14 +1,14 @@ from dataclasses import dataclass, asdict -from collections import defaultdict -from typing import Dict, List, Optional +from typing import cast, List, Optional +from hexbytes.main import HexBytes + +from eth_typing.encoding import HexStr +from tqdm import tqdm from web3 import Web3 -import web3 -from web3.types import FilterParams +from web3.types import FilterParams, LogReceipt from web3._utils.events import get_event_data -w3 = Web3(Web3.HTTPProvider("http://127.0.0.1:18375")) - # First abi is for old NFT's like crypto kitties # The erc721 standart requieres that Transfer event is indexed for all arguments # That is how we get distinguished from erc20 transfer events @@ -89,7 +89,7 @@ class NFT_contract: total_supply: str -def get_erc721_contract_info(address: str) -> NFT_contract: +def get_erc721_contract_info(w3: Web3, address: str) -> NFT_contract: contract = w3.eth.contract( address=w3.toChecksumAddress(address), abi=erc721_functions_abi ) @@ -101,7 +101,10 @@ def get_erc721_contract_info(address: str) -> NFT_contract: ) -transfer_event_signature = w3.sha3(text="Transfer(address,address,uint256)").hex() +# SHA3 hash of the string "Transfer(address,address,uint256)" +TRANSFER_EVENT_SIGNATURE = HexBytes( + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" +) @dataclass @@ -110,7 +113,7 @@ class NFTTransferRaw: transfer_from: str transfer_to: str tokenId: int - transfer_tx: bytes + transfer_tx: HexBytes @dataclass @@ -124,14 +127,14 @@ class NFTTransfer: is_mint: bool = False -def get_value_by_tx(tx_hash): +def get_value_by_tx(w3: Web3, tx_hash: HexBytes): print(f"Trying to get tx: {tx_hash.hex()}") tx = w3.eth.get_transaction(tx_hash) print("got it") return tx["value"] -def decode_nft_transfer_data(log) -> Optional[NFTTransferRaw]: +def decode_nft_transfer_data(w3: Web3, log: LogReceipt) -> Optional[NFTTransferRaw]: for abi in erc721_transfer_event_abis: try: transfer_data = get_event_data(w3.codec, abi, log) @@ -149,47 +152,34 @@ def decode_nft_transfer_data(log) -> Optional[NFTTransferRaw]: def get_nft_transfers( - block_number_from: int, contract_address: Optional[str] = None + w3: Web3, + from_block: Optional[int] = None, + to_block: Optional[int] = None, + contract_address: Optional[str] = None, ) -> List[NFTTransfer]: - filter_params = FilterParams( - fromBlock=block_number_from, topics=[transfer_event_signature] - ) + filter_params = FilterParams(topics=[cast(HexStr, TRANSFER_EVENT_SIGNATURE.hex())]) + + if from_block is not None: + filter_params["fromBlock"] = from_block + + if to_block is not None: + filter_params["toBlock"] = to_block if contract_address is not None: filter_params["address"] = w3.toChecksumAddress(contract_address) logs = w3.eth.get_logs(filter_params) nft_transfers: List[NFTTransfer] = [] - tx_value: Dict[bytes, List[NFTTransferRaw]] = defaultdict(list) - for log in logs: - nft_transfer = decode_nft_transfer_data(log) + for log in tqdm(logs): + nft_transfer = decode_nft_transfer_data(w3, log) if nft_transfer is not None: - tx_value[nft_transfer.transfer_tx].append(nft_transfer) - - for tx_hash, transfers in tx_value.items(): - # value = get_value_by_tx(tx_hash) - value = 0 - for transfer in transfers: kwargs = { - **asdict(transfer), - "transfer_tx": transfer.transfer_tx.hex(), - "is_mint": transfer.transfer_from + **asdict(nft_transfer), + "transfer_tx": nft_transfer.transfer_tx.hex(), + "is_mint": nft_transfer.transfer_from == "0x0000000000000000000000000000000000000000", - "value": value, } - parsed_transfer = NFTTransfer(**kwargs) + parsed_transfer = NFTTransfer(**kwargs) # type: ignore nft_transfers.append(parsed_transfer) return nft_transfers - - -cryptoKittiesAddress = "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d" -transfesrs = get_nft_transfers( - w3.eth.block_number - 120, -) - -print(transfesrs) -print(f"Total nft transfers: {len(transfesrs)}") -minted = list(filter(lambda transfer: transfer.is_mint == True, transfesrs)) -# print(minted) -print(f"Minted count: {len(minted)}") diff --git a/crawlers/mooncrawl/mooncrawl/ethcrawler.py b/crawlers/mooncrawl/mooncrawl/ethcrawler.py index 793ab3ea..c19079a2 100644 --- a/crawlers/mooncrawl/mooncrawl/ethcrawler.py +++ b/crawlers/mooncrawl/mooncrawl/ethcrawler.py @@ -8,11 +8,13 @@ import json import os import sys import time -from typing import Iterator, List +from typing import cast, Iterator, List import dateutil.parser +from web3 import Web3 from .ethereum import ( + connect, crawl_blocks_executor, crawl_blocks, check_missing_blocks, @@ -21,8 +23,9 @@ from .ethereum import ( DateRange, trending, ) +from .eth_nft_explorer import get_nft_transfers from .publish import publish_json -from .settings import MOONSTREAM_CRAWL_WORKERS +from .settings import MOONSTREAM_CRAWL_WORKERS, MOONSTREAM_IPC_PATH from .version import MOONCRAWL_VERSION @@ -31,6 +34,22 @@ class ProcessingOrder(Enum): ASCENDING = 1 +def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3: + """ + Returns a web3 client either by parsing "--web3" argument on the given arguments or by looking up + the MOONSTREAM_IPC_PATH environment variable. + """ + web3_connection_string = MOONSTREAM_IPC_PATH + args_web3 = vars(args).get("web3") + if args_web3 is not None: + web3_connection_string = cast(str, args_web3) + if web3_connection_string is None: + raise ValueError( + "Could not find Web3 connection information in arguments or in MOONSTREAM_IPC_PATH environment variable" + ) + return connect(web3_connection_string) + + def yield_blocks_numbers_lists( blocks_range_str: str, order: ProcessingOrder = ProcessingOrder.DESCENDING, @@ -200,6 +219,16 @@ def ethcrawler_trending_handler(args: argparse.Namespace) -> None: json.dump(results, ofp) +def ethcrawler_nft_handler(args: argparse.Namespace) -> None: + web3_client = web3_client_from_cli_or_env(args) + transfers = get_nft_transfers(web3_client, args.start, args.end, args.address) + for transfer in transfers: + print(transfer) + + print("Total transfers:", len(transfers)) + print("Mints:", len([transfer for transfer in transfers if transfer.is_mint])) + + def main() -> None: parser = argparse.ArgumentParser(description="Moonstream crawlers CLI") parser.set_defaults(func=lambda _: parser.print_help()) @@ -389,6 +418,38 @@ def main() -> None: ) parser_ethcrawler_trending.set_defaults(func=ethcrawler_trending_handler) + parser_ethcrawler_nft = subcommands.add_parser( + "nft", description="Collect information about NFTs from Ethereum blockchains" + ) + parser_ethcrawler_nft.add_argument( + "-s", + "--start", + type=int, + default=None, + help="Starting block number (inclusive if block available)", + ) + parser_ethcrawler_nft.add_argument( + "-e", + "--end", + type=int, + default=None, + help="Ending block number (inclusive if block available)", + ) + parser_ethcrawler_nft.add_argument( + "-a", + "--address", + type=str, + default=None, + help="(Optional) NFT contract address that you want to limit the crawl to, e.g. 0x06012c8cf97BEaD5deAe237070F9587f8E7A266d for CryptoKitties.", + ) + parser_ethcrawler_nft.add_argument( + "--web3", + type=str, + default=None, + help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_IPC_PATH environment variable.", + ) + parser_ethcrawler_nft.set_defaults(func=ethcrawler_nft_handler) + args = parser.parse_args() args.func(args) diff --git a/crawlers/mooncrawl/mypy.ini b/crawlers/mooncrawl/mypy.ini index 47838c47..45381262 100644 --- a/crawlers/mooncrawl/mypy.ini +++ b/crawlers/mooncrawl/mypy.ini @@ -8,3 +8,6 @@ ignore_missing_imports = True [mypy-pyevmasm.*] ignore_missing_imports = True + +[mypy-tqdm.*] +ignore_missing_imports = True