From 19c0dfeeb39e1c3691aa9a9e708970f3c5fe7132 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Fri, 3 Sep 2021 14:10:13 -0700 Subject: [PATCH] [WIP] Working prototypes for "nft ethereum label" and "nft ethereum summary" TODO: - `nft ethereum label` should have reasonable defaults for `--start` and `--end`. It should use the currenty block for `--end` and the last labelled block for `--start`? - Keep on plugging away at `nft ethereum summary` as per: https://github.com/bugout-dev/moonstream/issues/223 --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 73 +++++++--------- crawlers/mooncrawl/mooncrawl/nft/ethereum.py | 87 ++++++++++++++------ 2 files changed, 91 insertions(+), 69 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index e000a47e..6fe0224d 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -2,11 +2,11 @@ A command line tool to crawl information about NFTs from various sources. """ import argparse -import json -import os +from datetime import datetime, timedelta, timezone import sys from typing import cast +import dateutil.parser from moonstreamdb.db import yield_db_session_ctx from web3 import Web3 @@ -40,32 +40,34 @@ def ethereum_label_handler(args: argparse.Namespace) -> None: def ethereum_summary_handler(args: argparse.Namespace) -> None: - web3_client = web3_client_from_cli_or_env(args) - result = ethereum_summary(web3_client, args.start, args.end, args.address) + with yield_db_session_ctx() as db_session: + result = ethereum_summary(db_session, args.start, args.end) - start_time = result.get("date_range", {}).get("start_time", "UNKNOWN") - start_block = result.get("blocks", {}).get("start", -1) - end_time = result.get("date_range", {}).get("end_time", "UNKNOWN") - end_block = result.get("blocks", {}).get("end", -1) + # start_time = result.get("date_range", {}).get("start_time", "UNKNOWN") + # start_block = result.get("blocks", {}).get("start", -1) + # end_time = result.get("date_range", {}).get("end_time", "UNKNOWN") + # end_block = result.get("blocks", {}).get("end", -1) - humbug_token = args.humbug - if humbug_token is None: - humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") - if humbug_token: - title = f"NFT activity on the Ethereum blockchain: {start_time} (block {start_block}) to {end_time} (block {end_block})" - publish_json( - "nft_ethereum", - humbug_token, - title, - result, - tags=[f"crawler_version:{MOONCRAWL_VERSION}"], - wait=False, - ) - with args.outfile as ofp: - json.dump(result, ofp) + # humbug_token = args.humbug + # if humbug_token is None: + # humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") + # if humbug_token: + # title = f"NFT activity on the Ethereum blockchain: {start_time} (block {start_block}) to {end_time} (block {end_block})" + # publish_json( + # "nft_ethereum", + # humbug_token, + # title, + # result, + # tags=[f"crawler_version:{MOONCRAWL_VERSION}"], + # wait=False, + # ) + # with args.outfile as ofp: + # json.dump(result, ofp) def main() -> None: + time_now = datetime.now(timezone.utc) + parser = argparse.ArgumentParser(description="Moonstream NFT crawlers") parser.set_defaults(func=lambda _: parser.print_help()) subcommands = parser.add_subparsers(description="Subcommands") @@ -116,29 +118,16 @@ def main() -> None: parser_ethereum_summary.add_argument( "-s", "--start", - type=int, - default=None, - help="Starting block number (inclusive if block available)", + type=dateutil.parser.parse, + default=(time_now - timedelta(hours=1, minutes=0)).isoformat(), + help=f"Start time for window to calculate NFT statistics (default: {(time_now - timedelta(hours=1,minutes=0)).isoformat()})", ) parser_ethereum_summary.add_argument( "-e", "--end", - type=int, - default=None, - help="Ending block number (inclusive if block available)", - ) - parser_ethereum_summary.add_argument( - "-a", - "--address", - type=str, - default=None, - help="(Optional) NFT contract address that you want to limit the crawl to, e.g. 0x06012c8cf97BEaD5deAe237070F9587f8E7A266d for CryptoKitties.", - ) - parser_ethereum_summary.add_argument( - "--web3", - type=str, - default=None, - help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_IPC_PATH environment variable.", + type=dateutil.parser.parse, + default=time_now.isoformat(), + help=f"End time for window to calculate NFT statistics (default: {time_now.isoformat()})", ) parser_ethereum_summary.add_argument( "--humbug", diff --git a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py index 78fe235f..dfe0bac1 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py @@ -5,7 +5,13 @@ from hexbytes.main import HexBytes from typing import Any, cast, Dict, List, Optional, Set, Tuple from eth_typing.encoding import HexStr -from moonstreamdb.models import EthereumAddress, EthereumLabel, EthereumTransaction +from moonstreamdb.models import ( + EthereumAddress, + EthereumBlock, + EthereumLabel, + EthereumTransaction, +) +from sqlalchemy import and_ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session from tqdm import tqdm @@ -13,6 +19,8 @@ from web3 import Web3 from web3.types import FilterParams, LogReceipt from web3._utils.events import get_event_data +from ..ethereum import DateRange + # Default length (in blocks) of an Ethereum NFT crawl. DEFAULT_CRAWL_LENGTH = 100 @@ -454,33 +462,58 @@ def add_labels( def summary( - w3: Web3, - from_block: Optional[int] = None, - to_block: Optional[int] = None, - address: Optional[str] = None, + db_session: Session, + start_time: datetime, + end_time: datetime, ) -> Dict[str, Any]: - start, end = get_block_bounds(w3, from_block, to_block) - start_block = w3.eth.get_block(start) - start_time = datetime.utcfromtimestamp(start_block["timestamp"]).isoformat() - end_block = w3.eth.get_block(end) - end_time = datetime.utcfromtimestamp(end_block["timestamp"]).isoformat() + start_timestamp = int(start_time.timestamp()) + end_timestamp = int(end_time.timestamp()) - transfers = get_nft_transfers(w3, start, end, address) - num_mints = sum(transfer.is_mint for transfer in transfers) + base_query = ( + db_session.query( + EthereumLabel.label, + EthereumLabel.label_data, + EthereumLabel.address_id, + EthereumTransaction.hash, + EthereumTransaction.value, + EthereumBlock.block_number, + EthereumBlock.timestamp, + ) + .join( + EthereumTransaction, + EthereumLabel.transaction_hash == EthereumTransaction.hash, + ) + .join( + EthereumBlock, + EthereumTransaction.block_number == EthereumBlock.block_number, + ) + .filter( + and_( + EthereumBlock.timestamp >= start_timestamp, + EthereumBlock.timestamp <= end_timestamp, + ) + ) + .filter(EthereumLabel.label.in_([MINT_LABEL, TRANSFER_LABEL])) + ) - result = { - "date_range": { - "start_time": start_time, - "include_start": True, - "end_time": end_time, - "include_end": True, - }, - "blocks": { - "start": start, - "end": end, - }, - "num_transfers": len(transfers), - "num_mints": num_mints, - } + print(base_query.distinct(EthereumTransaction.hash).count()) - return result + return {} + + +# result = { +# "date_range": { +# "start_time": start_time, +# "include_start": True, +# "end_time": end_time, +# "include_end": True, +# }, +# "blocks": { +# "start": start, +# "end": end, +# }, +# "num_transfers": len(transfers), +# "num_mints": num_mints, +# } + +# return result