kopia lustrzana https://github.com/bugout-dev/moonstream
[WIP] Working prototypes for "nft ethereum label" and "nft ethereum summary"
TODO: - `nft ethereum label` should have reasonable defaults for `--start` and `--end`. It should use the currenty block for `--end` and the last labelled block for `--start`? - Keep on plugging away at `nft ethereum summary` as per: https://github.com/bugout-dev/moonstream/issues/223pull/226/head
Neeraj Kashyap
rodzic
d0cbff9277
commit
19c0dfeeb3
|
|
@ -2,11 +2,11 @@
|
|||
A command line tool to crawl information about NFTs from various sources.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import sys
|
||||
from typing import cast
|
||||
|
||||
import dateutil.parser
|
||||
from moonstreamdb.db import yield_db_session_ctx
|
||||
from web3 import Web3
|
||||
|
||||
|
|
@ -40,32 +40,34 @@ def ethereum_label_handler(args: argparse.Namespace) -> None:
|
|||
|
||||
|
||||
def ethereum_summary_handler(args: argparse.Namespace) -> None:
|
||||
web3_client = web3_client_from_cli_or_env(args)
|
||||
result = ethereum_summary(web3_client, args.start, args.end, args.address)
|
||||
with yield_db_session_ctx() as db_session:
|
||||
result = ethereum_summary(db_session, args.start, args.end)
|
||||
|
||||
start_time = result.get("date_range", {}).get("start_time", "UNKNOWN")
|
||||
start_block = result.get("blocks", {}).get("start", -1)
|
||||
end_time = result.get("date_range", {}).get("end_time", "UNKNOWN")
|
||||
end_block = result.get("blocks", {}).get("end", -1)
|
||||
# start_time = result.get("date_range", {}).get("start_time", "UNKNOWN")
|
||||
# start_block = result.get("blocks", {}).get("start", -1)
|
||||
# end_time = result.get("date_range", {}).get("end_time", "UNKNOWN")
|
||||
# end_block = result.get("blocks", {}).get("end", -1)
|
||||
|
||||
humbug_token = args.humbug
|
||||
if humbug_token is None:
|
||||
humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN")
|
||||
if humbug_token:
|
||||
title = f"NFT activity on the Ethereum blockchain: {start_time} (block {start_block}) to {end_time} (block {end_block})"
|
||||
publish_json(
|
||||
"nft_ethereum",
|
||||
humbug_token,
|
||||
title,
|
||||
result,
|
||||
tags=[f"crawler_version:{MOONCRAWL_VERSION}"],
|
||||
wait=False,
|
||||
)
|
||||
with args.outfile as ofp:
|
||||
json.dump(result, ofp)
|
||||
# humbug_token = args.humbug
|
||||
# if humbug_token is None:
|
||||
# humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN")
|
||||
# if humbug_token:
|
||||
# title = f"NFT activity on the Ethereum blockchain: {start_time} (block {start_block}) to {end_time} (block {end_block})"
|
||||
# publish_json(
|
||||
# "nft_ethereum",
|
||||
# humbug_token,
|
||||
# title,
|
||||
# result,
|
||||
# tags=[f"crawler_version:{MOONCRAWL_VERSION}"],
|
||||
# wait=False,
|
||||
# )
|
||||
# with args.outfile as ofp:
|
||||
# json.dump(result, ofp)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
time_now = datetime.now(timezone.utc)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Moonstream NFT crawlers")
|
||||
parser.set_defaults(func=lambda _: parser.print_help())
|
||||
subcommands = parser.add_subparsers(description="Subcommands")
|
||||
|
|
@ -116,29 +118,16 @@ def main() -> None:
|
|||
parser_ethereum_summary.add_argument(
|
||||
"-s",
|
||||
"--start",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Starting block number (inclusive if block available)",
|
||||
type=dateutil.parser.parse,
|
||||
default=(time_now - timedelta(hours=1, minutes=0)).isoformat(),
|
||||
help=f"Start time for window to calculate NFT statistics (default: {(time_now - timedelta(hours=1,minutes=0)).isoformat()})",
|
||||
)
|
||||
parser_ethereum_summary.add_argument(
|
||||
"-e",
|
||||
"--end",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Ending block number (inclusive if block available)",
|
||||
)
|
||||
parser_ethereum_summary.add_argument(
|
||||
"-a",
|
||||
"--address",
|
||||
type=str,
|
||||
default=None,
|
||||
help="(Optional) NFT contract address that you want to limit the crawl to, e.g. 0x06012c8cf97BEaD5deAe237070F9587f8E7A266d for CryptoKitties.",
|
||||
)
|
||||
parser_ethereum_summary.add_argument(
|
||||
"--web3",
|
||||
type=str,
|
||||
default=None,
|
||||
help="(Optional) Web3 connection string. If not provided, uses the value specified by MOONSTREAM_IPC_PATH environment variable.",
|
||||
type=dateutil.parser.parse,
|
||||
default=time_now.isoformat(),
|
||||
help=f"End time for window to calculate NFT statistics (default: {time_now.isoformat()})",
|
||||
)
|
||||
parser_ethereum_summary.add_argument(
|
||||
"--humbug",
|
||||
|
|
|
|||
|
|
@ -5,7 +5,13 @@ from hexbytes.main import HexBytes
|
|||
from typing import Any, cast, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from eth_typing.encoding import HexStr
|
||||
from moonstreamdb.models import EthereumAddress, EthereumLabel, EthereumTransaction
|
||||
from moonstreamdb.models import (
|
||||
EthereumAddress,
|
||||
EthereumBlock,
|
||||
EthereumLabel,
|
||||
EthereumTransaction,
|
||||
)
|
||||
from sqlalchemy import and_
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.orm import Session
|
||||
from tqdm import tqdm
|
||||
|
|
@ -13,6 +19,8 @@ from web3 import Web3
|
|||
from web3.types import FilterParams, LogReceipt
|
||||
from web3._utils.events import get_event_data
|
||||
|
||||
from ..ethereum import DateRange
|
||||
|
||||
# Default length (in blocks) of an Ethereum NFT crawl.
|
||||
DEFAULT_CRAWL_LENGTH = 100
|
||||
|
||||
|
|
@ -454,33 +462,58 @@ def add_labels(
|
|||
|
||||
|
||||
def summary(
|
||||
w3: Web3,
|
||||
from_block: Optional[int] = None,
|
||||
to_block: Optional[int] = None,
|
||||
address: Optional[str] = None,
|
||||
db_session: Session,
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
) -> Dict[str, Any]:
|
||||
start, end = get_block_bounds(w3, from_block, to_block)
|
||||
start_block = w3.eth.get_block(start)
|
||||
start_time = datetime.utcfromtimestamp(start_block["timestamp"]).isoformat()
|
||||
end_block = w3.eth.get_block(end)
|
||||
end_time = datetime.utcfromtimestamp(end_block["timestamp"]).isoformat()
|
||||
start_timestamp = int(start_time.timestamp())
|
||||
end_timestamp = int(end_time.timestamp())
|
||||
|
||||
transfers = get_nft_transfers(w3, start, end, address)
|
||||
num_mints = sum(transfer.is_mint for transfer in transfers)
|
||||
base_query = (
|
||||
db_session.query(
|
||||
EthereumLabel.label,
|
||||
EthereumLabel.label_data,
|
||||
EthereumLabel.address_id,
|
||||
EthereumTransaction.hash,
|
||||
EthereumTransaction.value,
|
||||
EthereumBlock.block_number,
|
||||
EthereumBlock.timestamp,
|
||||
)
|
||||
.join(
|
||||
EthereumTransaction,
|
||||
EthereumLabel.transaction_hash == EthereumTransaction.hash,
|
||||
)
|
||||
.join(
|
||||
EthereumBlock,
|
||||
EthereumTransaction.block_number == EthereumBlock.block_number,
|
||||
)
|
||||
.filter(
|
||||
and_(
|
||||
EthereumBlock.timestamp >= start_timestamp,
|
||||
EthereumBlock.timestamp <= end_timestamp,
|
||||
)
|
||||
)
|
||||
.filter(EthereumLabel.label.in_([MINT_LABEL, TRANSFER_LABEL]))
|
||||
)
|
||||
|
||||
result = {
|
||||
"date_range": {
|
||||
"start_time": start_time,
|
||||
"include_start": True,
|
||||
"end_time": end_time,
|
||||
"include_end": True,
|
||||
},
|
||||
"blocks": {
|
||||
"start": start,
|
||||
"end": end,
|
||||
},
|
||||
"num_transfers": len(transfers),
|
||||
"num_mints": num_mints,
|
||||
}
|
||||
print(base_query.distinct(EthereumTransaction.hash).count())
|
||||
|
||||
return result
|
||||
return {}
|
||||
|
||||
|
||||
# result = {
|
||||
# "date_range": {
|
||||
# "start_time": start_time,
|
||||
# "include_start": True,
|
||||
# "end_time": end_time,
|
||||
# "include_end": True,
|
||||
# },
|
||||
# "blocks": {
|
||||
# "start": start,
|
||||
# "end": end,
|
||||
# },
|
||||
# "num_transfers": len(transfers),
|
||||
# "num_mints": num_mints,
|
||||
# }
|
||||
|
||||
# return result
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue