mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Cleaned up NFT crawler code a bit 

Hooked it up to comamnd line
pull/226/head
Neeraj Kashyap 2021-09-01 15:01:24 -07:00
rodzic a80898a6a9
commit 738bc20b5e
4 zmienionych plików z 99 dodań i 43 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
crawlers/mooncrawl/.gitignore vendored 100644
Wyświetl plik

@ -0,0 +1,2 @@
.venv/
.mooncrawl/

72
crawlers/mooncrawl/mooncrawl/eth_nft_explorer.py
Wyświetl plik

@ -1,14 +1,14 @@
from dataclasses import dataclass, asdict
from collections import defaultdict
from typing import Dict, List, Optional
from typing import cast, List, Optional
from hexbytes.main import HexBytes
from eth_typing.encoding import HexStr
from tqdm import tqdm
from web3 import Web3
import web3
from web3.types import FilterParams
from web3.types import FilterParams, LogReceipt
from web3._utils.events import get_event_data
w3 = Web3(Web3.HTTPProvider("http://127.0.0.1:18375"))
# First abi is for old NFT's like crypto kitties
# The erc721 standart requieres that Transfer event is indexed for all arguments
# That is how we get distinguished from erc20 transfer events
@ -89,7 +89,7 @@ class NFT_contract:
total_supply: str
def get_erc721_contract_info(address: str) -> NFT_contract:
def get_erc721_contract_info(w3: Web3, address: str) -> NFT_contract:
contract = w3.eth.contract(
address=w3.toChecksumAddress(address), abi=erc721_functions_abi
)
@ -101,7 +101,10 @@ def get_erc721_contract_info(address: str) -> NFT_contract:
)
transfer_event_signature = w3.sha3(text="Transfer(address,address,uint256)").hex()
# SHA3 hash of the string "Transfer(address,address,uint256)"
TRANSFER_EVENT_SIGNATURE = HexBytes(
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
)
@dataclass
@ -110,7 +113,7 @@ class NFTTransferRaw:
transfer_from: str
transfer_to: str
tokenId: int
transfer_tx: bytes
transfer_tx: HexBytes
@dataclass
@ -124,14 +127,14 @@ class NFTTransfer:
is_mint: bool = False
def get_value_by_tx(tx_hash):
def get_value_by_tx(w3: Web3, tx_hash: HexBytes):
print(f"Trying to get tx: {tx_hash.hex()}")
tx = w3.eth.get_transaction(tx_hash)
print("got it")
return tx["value"]
def decode_nft_transfer_data(log) -> Optional[NFTTransferRaw]:
def decode_nft_transfer_data(w3: Web3, log: LogReceipt) -> Optional[NFTTransferRaw]:
for abi in erc721_transfer_event_abis:
try:
transfer_data = get_event_data(w3.codec, abi, log)
@ -149,47 +152,34 @@ def decode_nft_transfer_data(log) -> Optional[NFTTransferRaw]:
def get_nft_transfers(
block_number_from: int, contract_address: Optional[str] = None
w3: Web3,
from_block: Optional[int] = None,
to_block: Optional[int] = None,
contract_address: Optional[str] = None,
) -> List[NFTTransfer]:
filter_params = FilterParams(
fromBlock=block_number_from, topics=[transfer_event_signature]
)
filter_params = FilterParams(topics=[cast(HexStr, TRANSFER_EVENT_SIGNATURE.hex())])
if from_block is not None:
filter_params["fromBlock"] = from_block
if to_block is not None:
filter_params["toBlock"] = to_block
if contract_address is not None:
filter_params["address"] = w3.toChecksumAddress(contract_address)
logs = w3.eth.get_logs(filter_params)
nft_transfers: List[NFTTransfer] = []
tx_value: Dict[bytes, List[NFTTransferRaw]] = defaultdict(list)
for log in logs:
nft_transfer = decode_nft_transfer_data(log)
for log in tqdm(logs):
nft_transfer = decode_nft_transfer_data(w3, log)
if nft_transfer is not None:
tx_value[nft_transfer.transfer_tx].append(nft_transfer)
for tx_hash, transfers in tx_value.items():
# value = get_value_by_tx(tx_hash)
value = 0
for transfer in transfers:
kwargs = {
**asdict(transfer),
"transfer_tx": transfer.transfer_tx.hex(),
"is_mint": transfer.transfer_from
**asdict(nft_transfer),
"transfer_tx": nft_transfer.transfer_tx.hex(),
"is_mint": nft_transfer.transfer_from
== "0x0000000000000000000000000000000000000000",
"value": value,
}
parsed_transfer = NFTTransfer(**kwargs)
parsed_transfer = NFTTransfer(**kwargs) # type: ignore
nft_transfers.append(parsed_transfer)
return nft_transfers
cryptoKittiesAddress = "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d"
transfesrs = get_nft_transfers(
w3.eth.block_number - 120,
)
print(transfesrs)
print(f"Total nft transfers: {len(transfesrs)}")
minted = list(filter(lambda transfer: transfer.is_mint == True, transfesrs))
# print(minted)
print(f"Minted count: {len(minted)}")

65
crawlers/mooncrawl/mooncrawl/ethcrawler.py
Wyświetl plik

@ -8,11 +8,13 @@ import json
import os
import sys
import time
from typing import Iterator, List
from typing import cast, Iterator, List
import dateutil.parser
from web3 import Web3
from .ethereum import (
connect,
crawl_blocks_executor,
crawl_blocks,
check_missing_blocks,
@ -21,8 +23,9 @@ from .ethereum import (
DateRange,
trending,
)
from .eth_nft_explorer import get_nft_transfers
from .publish import publish_json
from .settings import MOONSTREAM_CRAWL_WORKERS
from .settings import MOONSTREAM_CRAWL_WORKERS, MOONSTREAM_IPC_PATH
from .version import MOONCRAWL_VERSION
@ -31,6 +34,22 @@ class ProcessingOrder(Enum):
ASCENDING = 1
def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3:
"""
Returns a web3 client either by parsing "--web3" argument on the given arguments or by looking up
the MOONSTREAM_IPC_PATH environment variable.
"""
web3_connection_string = MOONSTREAM_IPC_PATH
args_web3 = vars(args).get("web3")
if args_web3 is not None:
web3_connection_string = cast(str, args_web3)
if web3_connection_string is None:
raise ValueError(
"Could not find Web3 connection information in arguments or in MOONSTREAM_IPC_PATH environment variable"
)
return connect(web3_connection_string)
def yield_blocks_numbers_lists(
blocks_range_str: str,
order: ProcessingOrder = ProcessingOrder.DESCENDING,
@ -200,6 +219,16 @@ def ethcrawler_trending_handler(args: argparse.Namespace) -> None:
json.dump(results, ofp)
def ethcrawler_nft_handler(args: argparse.Namespace) -> None:
web3_client = web3_client_from_cli_or_env(args)
transfers = get_nft_transfers(web3_client, args.start, args.end, args.address)
for transfer in transfers:
print(transfer)
print("Total transfers:", len(transfers))
print("Mints:", len([transfer for transfer in transfers if transfer.is_mint]))
def main() -> None:
parser = argparse.ArgumentParser(description="Moonstream crawlers CLI")
parser.set_defaults(func=lambda _: parser.print_help())
@ -389,6 +418,38 @@ def main() -> None:
)
parser_ethcrawler_trending.set_defaults(func=ethcrawler_trending_handler)
parser_ethcrawler_nft = subcommands.add_parser(
"nft", description="Collect information about NFTs from Ethereum blockchains"
)
parser_ethcrawler_nft.add_argument(
"-s",
"--start",
type=int,
default=None,
help="Starting block number (inclusive if block available)",
)
parser_ethcrawler_nft.add_argument(
"-e",
"--end",
type=int,
default=None,
help="Ending block number (inclusive if block available)",
)
parser_ethcrawler_nft.add_argument(
"-a",
"--address",
type=str,
default=None,
help="(Optional) NFT contract address that you want to limit the crawl to, e.g. 0x06012c8cf97BEaD5deAe237070F9587f8E7A266d for CryptoKitties.",
)
parser_ethcrawler_nft.add_argument(
"--web3",
type=str,
default=None,
help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_IPC_PATH environment variable.",
)
parser_ethcrawler_nft.set_defaults(func=ethcrawler_nft_handler)
args = parser.parse_args()
args.func(args)

3
crawlers/mooncrawl/mypy.ini
Wyświetl plik

@ -8,3 +8,6 @@ ignore_missing_imports = True
[mypy-pyevmasm.*]
ignore_missing_imports = True
[mypy-tqdm.*]
ignore_missing_imports = True