diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index 6fe0224d..514035b6 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -3,6 +3,7 @@ A command line tool to crawl information about NFTs from various sources. """ import argparse from datetime import datetime, timedelta, timezone +import json import sys from typing import cast @@ -43,11 +44,6 @@ def ethereum_summary_handler(args: argparse.Namespace) -> None: with yield_db_session_ctx() as db_session: result = ethereum_summary(db_session, args.start, args.end) - # start_time = result.get("date_range", {}).get("start_time", "UNKNOWN") - # start_block = result.get("blocks", {}).get("start", -1) - # end_time = result.get("date_range", {}).get("end_time", "UNKNOWN") - # end_block = result.get("blocks", {}).get("end", -1) - # humbug_token = args.humbug # if humbug_token is None: # humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") @@ -61,8 +57,8 @@ def ethereum_summary_handler(args: argparse.Namespace) -> None: # tags=[f"crawler_version:{MOONCRAWL_VERSION}"], # wait=False, # ) - # with args.outfile as ofp: - # json.dump(result, ofp) + with args.outfile as ofp: + json.dump(result, ofp) def main() -> None: diff --git a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py index dfe0bac1..1a77bf59 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py @@ -11,9 +11,9 @@ from moonstreamdb.models import ( EthereumLabel, EthereumTransaction, ) -from sqlalchemy import and_ +from sqlalchemy import and_, func from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, Query from tqdm import tqdm from web3 import Web3 from web3.types import FilterParams, LogReceipt @@ -28,7 +28,6 @@ NFT_LABEL = "erc721" MINT_LABEL = "nft_mint" TRANSFER_LABEL = "nft_transfer" - logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -469,51 +468,94 @@ def summary( start_timestamp = int(start_time.timestamp()) end_timestamp = int(end_time.timestamp()) - base_query = ( - db_session.query( - EthereumLabel.label, - EthereumLabel.label_data, - EthereumLabel.address_id, - EthereumTransaction.hash, - EthereumTransaction.value, - EthereumBlock.block_number, - EthereumBlock.timestamp, - ) - .join( - EthereumTransaction, - EthereumLabel.transaction_hash == EthereumTransaction.hash, - ) + time_filter = and_( + EthereumBlock.timestamp >= start_timestamp, + EthereumBlock.timestamp <= end_timestamp, + ) + + transactions_query = ( + db_session.query(EthereumTransaction) .join( EthereumBlock, EthereumTransaction.block_number == EthereumBlock.block_number, ) - .filter( - and_( - EthereumBlock.timestamp >= start_timestamp, - EthereumBlock.timestamp <= end_timestamp, - ) - ) - .filter(EthereumLabel.label.in_([MINT_LABEL, TRANSFER_LABEL])) + .filter(time_filter) ) - print(base_query.distinct(EthereumTransaction.hash).count()) + def nft_query(label: str) -> Query: + query = ( + db_session.query( + EthereumLabel.label, + EthereumLabel.label_data, + EthereumLabel.address_id, + EthereumTransaction.hash, + EthereumTransaction.value, + EthereumBlock.block_number, + EthereumBlock.timestamp, + ) + .join( + EthereumTransaction, + EthereumLabel.transaction_hash == EthereumTransaction.hash, + ) + .join( + EthereumBlock, + EthereumTransaction.block_number == EthereumBlock.block_number, + ) + .filter(time_filter) + .filter(EthereumLabel.label == label) + ) + return query - return {} + transfer_query = nft_query(TRANSFER_LABEL) + mint_query = nft_query(MINT_LABEL) + blocks_result: Dict[str, int] = {} + min_block = ( + db_session.query(func.min(EthereumBlock.block_number)) + .filter(time_filter) + .scalar() + ) + max_block = ( + db_session.query(func.max(EthereumBlock.block_number)) + .filter(time_filter) + .scalar() + ) + if min_block is not None: + blocks_result["start"] = min_block + if max_block is not None: + blocks_result["end"] = max_block -# result = { -# "date_range": { -# "start_time": start_time, -# "include_start": True, -# "end_time": end_time, -# "include_end": True, -# }, -# "blocks": { -# "start": start, -# "end": end, -# }, -# "num_transfers": len(transfers), -# "num_mints": num_mints, -# } + num_transactions = transactions_query.distinct(EthereumTransaction.hash).count() + num_transfers = transfer_query.distinct(EthereumTransaction.hash).count() -# return result + total_value = db_session.query( + func.sum(transactions_query.subquery().c.value) + ).scalar() + transfer_value = db_session.query( + func.sum(transfer_query.subquery().c.value) + ).scalar() + + num_minted = mint_query.distinct(EthereumTransaction.hash).count() + + result = { + "date_range": { + "start_time": start_time.isoformat(), + "include_start": True, + "end_time": end_time.isoformat(), + "include_end": True, + }, + "blocks": blocks_result, + "transactions": { + "total": f"{num_transactions}", + "amount": f"{num_transfers}", + "percentage": f"{num_transfers/num_transactions * 100}", + }, + "value": { + "total": f"{total_value}", + "amount": f"{transfer_value}", + "percentage": f"{transfer_value/total_value * 100}", + }, + "mints": num_minted, + } + + return result