diff --git a/crawlers/mooncrawl/mooncrawl/ethcrawler.py b/crawlers/mooncrawl/mooncrawl/ethcrawler.py index c19079a2..d9d51e5d 100644 --- a/crawlers/mooncrawl/mooncrawl/ethcrawler.py +++ b/crawlers/mooncrawl/mooncrawl/ethcrawler.py @@ -8,13 +8,11 @@ import json import os import sys import time -from typing import cast, Iterator, List +from typing import Iterator, List import dateutil.parser -from web3 import Web3 from .ethereum import ( - connect, crawl_blocks_executor, crawl_blocks, check_missing_blocks, @@ -23,7 +21,6 @@ from .ethereum import ( DateRange, trending, ) -from .eth_nft_explorer import get_nft_transfers from .publish import publish_json from .settings import MOONSTREAM_CRAWL_WORKERS, MOONSTREAM_IPC_PATH from .version import MOONCRAWL_VERSION @@ -34,22 +31,6 @@ class ProcessingOrder(Enum): ASCENDING = 1 -def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3: - """ - Returns a web3 client either by parsing "--web3" argument on the given arguments or by looking up - the MOONSTREAM_IPC_PATH environment variable. - """ - web3_connection_string = MOONSTREAM_IPC_PATH - args_web3 = vars(args).get("web3") - if args_web3 is not None: - web3_connection_string = cast(str, args_web3) - if web3_connection_string is None: - raise ValueError( - "Could not find Web3 connection information in arguments or in MOONSTREAM_IPC_PATH environment variable" - ) - return connect(web3_connection_string) - - def yield_blocks_numbers_lists( blocks_range_str: str, order: ProcessingOrder = ProcessingOrder.DESCENDING, @@ -219,16 +200,6 @@ def ethcrawler_trending_handler(args: argparse.Namespace) -> None: json.dump(results, ofp) -def ethcrawler_nft_handler(args: argparse.Namespace) -> None: - web3_client = web3_client_from_cli_or_env(args) - transfers = get_nft_transfers(web3_client, args.start, args.end, args.address) - for transfer in transfers: - print(transfer) - - print("Total transfers:", len(transfers)) - print("Mints:", len([transfer for transfer in transfers if transfer.is_mint])) - - def main() -> None: parser = argparse.ArgumentParser(description="Moonstream crawlers CLI") parser.set_defaults(func=lambda _: parser.print_help()) @@ -418,38 +389,6 @@ def main() -> None: ) parser_ethcrawler_trending.set_defaults(func=ethcrawler_trending_handler) - parser_ethcrawler_nft = subcommands.add_parser( - "nft", description="Collect information about NFTs from Ethereum blockchains" - ) - parser_ethcrawler_nft.add_argument( - "-s", - "--start", - type=int, - default=None, - help="Starting block number (inclusive if block available)", - ) - parser_ethcrawler_nft.add_argument( - "-e", - "--end", - type=int, - default=None, - help="Ending block number (inclusive if block available)", - ) - parser_ethcrawler_nft.add_argument( - "-a", - "--address", - type=str, - default=None, - help="(Optional) NFT contract address that you want to limit the crawl to, e.g. 0x06012c8cf97BEaD5deAe237070F9587f8E7A266d for CryptoKitties.", - ) - parser_ethcrawler_nft.add_argument( - "--web3", - type=str, - default=None, - help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_IPC_PATH environment variable.", - ) - parser_ethcrawler_nft.set_defaults(func=ethcrawler_nft_handler) - args = parser.parse_args() args.func(args) diff --git a/crawlers/mooncrawl/mooncrawl/nft/__init__.py b/crawlers/mooncrawl/mooncrawl/nft/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py new file mode 100644 index 00000000..96d74f89 --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -0,0 +1,83 @@ +""" +A command line tool to crawl information about NFTs from various sources. +""" +import argparse +from typing import cast + +from web3 import Web3 + +from ..ethereum import connect +from .ethereum import get_nft_transfers +from ..settings import MOONSTREAM_IPC_PATH + + +def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3: + """ + Returns a web3 client either by parsing "--web3" argument on the given arguments or by looking up + the MOONSTREAM_IPC_PATH environment variable. + """ + web3_connection_string = MOONSTREAM_IPC_PATH + args_web3 = vars(args).get("web3") + if args_web3 is not None: + web3_connection_string = cast(str, args_web3) + if web3_connection_string is None: + raise ValueError( + "Could not find Web3 connection information in arguments or in MOONSTREAM_IPC_PATH environment variable" + ) + return connect(web3_connection_string) + + +def ethereum_handler(args: argparse.Namespace) -> None: + web3_client = web3_client_from_cli_or_env(args) + transfers = get_nft_transfers(web3_client, args.start, args.end, args.address) + for transfer in transfers: + print(transfer) + + print("Total transfers:", len(transfers)) + print("Mints:", len([transfer for transfer in transfers if transfer.is_mint])) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Moonstream NFT crawlers") + parser.set_defaults(func=lambda _: parser.print_help()) + subcommands = parser.add_subparsers(description="Subcommands") + + parser_ethereum = subcommands.add_parser( + "ethereum", + description="Collect information about NFTs from Ethereum blockchains", + ) + parser_ethereum.add_argument( + "-s", + "--start", + type=int, + default=None, + help="Starting block number (inclusive if block available)", + ) + parser_ethereum.add_argument( + "-e", + "--end", + type=int, + default=None, + help="Ending block number (inclusive if block available)", + ) + parser_ethereum.add_argument( + "-a", + "--address", + type=str, + default=None, + help="(Optional) NFT contract address that you want to limit the crawl to, e.g. 0x06012c8cf97BEaD5deAe237070F9587f8E7A266d for CryptoKitties.", + ) + parser_ethereum.add_argument( + "--web3", + type=str, + default=None, + help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_IPC_PATH environment variable.", + ) + parser_ethereum.set_defaults(func=ethereum_handler) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/crawlers/mooncrawl/mooncrawl/eth_nft_explorer.py b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py similarity index 100% rename from crawlers/mooncrawl/mooncrawl/eth_nft_explorer.py rename to crawlers/mooncrawl/mooncrawl/nft/ethereum.py