mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #263 from bugout-dev/nft-explorer 

Improvements to "nft ethereum" crawlers
pull/278/head
Sergei Sumarokov 2021-09-23 13:15:35 +03:00 zatwierdzone przez GitHub
rodzic 49600ad1d6 20a7189706
commit 1d3c4c5d41
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
11 zmienionych plików z 462 dodań i 148 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

11
backend/moonstream/api.py
Wyświetl plik

@ -8,12 +8,12 @@ from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from . import data
from .routes.subscriptions import app as subscriptions_api
from .routes.users import app as users_api
from .routes.txinfo import app as txinfo_api
from .routes.streams import app as streams_api
from .routes.address_info import app as addressinfo_api
from .routes.nft import app as nft_api
from .routes.subscriptions import app as subscriptions_api
from .routes.streams import app as streams_api
from .routes.txinfo import app as txinfo_api
from .routes.users import app as users_api
from .settings import ORIGINS
from .version import MOONSTREAM_VERSION
@ -51,3 +51,4 @@ app.mount("/users", users_api)
app.mount("/streams", streams_api)
app.mount("/txinfo", txinfo_api)
app.mount("/address_info", addressinfo_api)
app.mount("/nft", nft_api)

46
backend/moonstream/providers/bugout.py
Wyświetl plik

@ -321,6 +321,34 @@ class EthereumTXPoolProvider(BugoutEventProvider):
return subscriptions_filters
class NftProvider(BugoutEventProvider):
def __init__(
self,
event_type: str,
description: str,
default_time_interval_seconds: int,
estimated_events_per_time_interval: float,
tags: Optional[List[str]] = None,
batch_size: int = 100,
timeout: float = 30.0,
):
super().__init__(
event_type=event_type,
description=description,
default_time_interval_seconds=default_time_interval_seconds,
estimated_events_per_time_interval=estimated_events_per_time_interval,
tags=tags,
batch_size=batch_size,
timeout=timeout,
)
def parse_filters(
self, query: StreamQuery, user_subscriptions: Dict[str, List[BugoutResource]]
) -> Optional[List[str]]:
return []
whalewatch_description = """Event provider for Ethereum whale watch.
Shows the top 10 addresses active on the Ethereum blockchain over the last hour in the following categories:
@ -350,3 +378,21 @@ ethereum_txpool_provider = EthereumTXPoolProvider(
estimated_events_per_time_interval=50,
tags=["client:ethereum-txpool-crawler-0"],
)
nft_summary_description = """Event provider for NFT market summaries.
This provider periodically generates NFT market summaries for the last hour of market activity.
Currently, it summarizes the activities on the following NFT markets:
1. The Ethereum market
This provider is currently not accessible for subscription. The data from this provider is publicly
available at the /nft endpoint."""
nft_summary_provider = NftProvider(
event_type="nft_summary",
description=nft_summary_description,
# 40 blocks per summary, 15 seconds per block + 2 seconds wiggle room.
default_time_interval_seconds=40 * 17,
estimated_events_per_time_interval=1,
tags=["crawl_type:nft_ethereum"],
)

85
backend/moonstream/routes/nft.py 100644
Wyświetl plik

@ -0,0 +1,85 @@
"""
Moonstream's /nft endpoints.
These endpoints provide public access to NFT market summaries. No authentication required.
"""
from datetime import datetime
import logging
from typing import Optional
from bugout.data import BugoutResource
from fastapi import Depends, FastAPI, Query
from moonstreamdb import db
from fastapi.middleware.cors import CORSMiddleware
from sqlalchemy.orm import Session
from .. import data
from ..providers.bugout import nft_summary_provider
from ..settings import (
bugout_client,
DOCS_TARGET_PATH,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
ORIGINS,
)
from ..stream_queries import StreamQuery
from ..version import MOONSTREAM_VERSION
logger = logging.getLogger(__name__)
tags_metadata = [
{"name": "nft", "description": "NFT market summaries"},
]
app = FastAPI(
title=f"Moonstream /nft API",
description="User, token and password handlers.",
version=MOONSTREAM_VERSION,
openapi_tags=tags_metadata,
openapi_url="/openapi.json",
docs_url=None,
redoc_url=f"/{DOCS_TARGET_PATH}",
)
app.add_middleware(
CORSMiddleware,
allow_origins=ORIGINS,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
@app.get("/", tags=["streams"], response_model=data.GetEventsResponse)
async def stream_handler(
start_time: int = Query(0),
end_time: Optional[int] = Query(None),
include_start: bool = Query(False),
include_end: bool = Query(False),
db_session: Session = Depends(db.yield_db_session),
) -> data.GetEventsResponse:
stream_boundary = data.StreamBoundary(
start_time=start_time,
end_time=end_time,
include_start=include_start,
include_end=include_end,
)
result = nft_summary_provider.get_events(
db_session=db_session,
bugout_client=bugout_client,
data_journal_id=MOONSTREAM_DATA_JOURNAL_ID,
data_access_token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
stream_boundary=stream_boundary,
user_subscriptions={nft_summary_provider.event_type: []},
query=StreamQuery(subscription_types=[nft_summary_provider.event_type]),
)
if result is None:
return data.GetEventsResponse(stream_boundary=stream_boundary, events=[])
provider_stream_boundary, events = result
return data.GetEventsResponse(
stream_boundary=provider_stream_boundary, events=events
)

2
backend/moonstream/routes/txinfo.py
Wyświetl plik

@ -28,7 +28,7 @@ tags_metadata = [
]
app = FastAPI(
title=f"Moonstream users API.",
title=f"Moonstream /txinfo API.",
description="User, token and password handlers.",
version=MOONSTREAM_VERSION,
openapi_tags=tags_metadata,

293
crawlers/mooncrawl/mooncrawl/nft/cli.py
Wyświetl plik

@ -2,25 +2,46 @@
A command line tool to crawl information about NFTs from various sources.
"""
import argparse
from datetime import datetime, timedelta, timezone
from datetime import datetime, timezone
import json
import os
import logging
import sys
import time
from typing import Any, Dict, cast
from typing import Any, cast, Dict, Optional
import dateutil.parser
from bugout.app import Bugout
from bugout.journal import SearchOrder
from moonstreamdb.db import yield_db_session_ctx
from moonstreamdb.models import EthereumBlock
from moonstreamdb.models import EthereumBlock, EthereumTransaction, EthereumLabel
from sqlalchemy.orm.session import Session
from web3 import Web3
from ..ethereum import connect
from .ethereum import summary as ethereum_summary, add_labels
from .ethereum import (
summary as ethereum_summary,
add_labels,
MINT_LABEL,
TRANSFER_LABEL,
SUMMARY_KEY_ARGS,
SUMMARY_KEY_ID,
SUMMARY_KEY_NUM_BLOCKS,
SUMMARY_KEY_START_BLOCK,
SUMMARY_KEY_END_BLOCK,
)
from ..publish import publish_json
from ..settings import MOONSTREAM_IPC_PATH
from ..settings import (
MOONSTREAM_IPC_PATH,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
NFT_HUMBUG_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
)
from ..version import MOONCRAWL_VERSION
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
BLOCKS_PER_SUMMARY = 40
def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3:
"""
@ -47,48 +68,156 @@ def get_latest_block_from_db(db_session: Session):
)
def ethereum_sync_handler(args: argparse.Namespace) -> None:
web3_client = web3_client_from_cli_or_env(args)
humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN")
if humbug_token is None:
raise ValueError("MOONSTREAM_HUMBUG_TOKEN env variable is not set")
def get_latest_summary_block() -> Optional[int]:
try:
with yield_db_session_ctx() as db_session:
start = args.start
bugout_client = Bugout()
query = "#crawl_type:nft_ethereum"
events = bugout_client.search(
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_DATA_JOURNAL_ID,
query,
limit=1,
timeout=30.0,
order=SearchOrder.DESCENDING,
)
if not events.results:
logger.warning("There is no summaries in Bugout")
return None
last_event = events.results[0]
content = cast(str, last_event.content)
return json.loads(content)["end_block"]
except Exception as e:
logger.error(f"Failed to get summary from Bugout : {e}")
return None
def get_latest_nft_labeled_block(db_session: Session) -> Optional[int]:
query = (
db_session.query(
EthereumLabel.label,
EthereumTransaction.hash,
EthereumBlock.block_number,
)
.join(
EthereumTransaction,
EthereumLabel.transaction_hash == EthereumTransaction.hash,
)
.join(
EthereumBlock,
EthereumTransaction.block_number == EthereumBlock.block_number,
)
.filter(EthereumLabel.label.in_([MINT_LABEL, TRANSFER_LABEL]))
.order_by(EthereumBlock.block_number.desc())
.limit(1)
)
start_block = query.one_or_none()
if start_block is not None:
return start_block.block_number
else:
return None
def sync_labels(db_session: Session, web3_client: Web3, start: Optional[int]) -> int:
if start is None:
logger.info(
"Syncing label start block is not given, getting it from latest nft label in db"
)
start = get_latest_nft_labeled_block(db_session)
if start is None:
time_now = datetime.now(timezone.utc)
week_ago = time_now - timedelta(weeks=1)
logger.warning(
"Didn't find any nft labels in db, starting sync from 1st Jan 2021 before now"
)
start_date = datetime(2021, 1, 1, tzinfo=timezone.utc)
start = (
db_session.query(EthereumBlock)
.filter(EthereumBlock.timestamp >= week_ago.timestamp())
.filter(EthereumBlock.timestamp >= start_date.timestamp())
.order_by(EthereumBlock.timestamp.asc())
.limit(1)
.one()
).block_number
logger.info(f"Syncing labels, start block: {start}")
latest_block = get_latest_block_from_db(db_session)
end = latest_block.block_number
if start > end:
logger.warn(f"Start block {start} is greater than latest_block {end} in db")
logger.warn("Maybe ethcrawler is not syncing or nft sync is up to date")
return start - 1
logger.info(f"Labeling blocks {start}-{end}")
add_labels(web3_client, db_session, start, end)
return latest_block.block_number
def sync_summaries(
db_session: Session,
start: Optional[int],
end: int,
) -> int:
if start is None:
logger.info(
"Syncing summary start block is not given, getting it from latest nft summary from Bugout"
)
start = get_latest_summary_block()
if start is not None:
start += 1
else:
logger.info(
"There is no entry in Bugout, starting to create summaries from 1st Jan 2021"
)
start_date = datetime(2021, 1, 1, tzinfo=timezone.utc)
start = (
db_session.query(EthereumBlock)
.filter(EthereumBlock.timestamp >= start_date.timestamp())
.order_by(EthereumBlock.timestamp.asc())
.limit(1)
.one()
).block_number
latest_block = get_latest_block_from_db(db_session)
end = latest_block.block_number
assert (
start <= end
), f"Start block {start} is greater than latest_block {end} in db"
logger.info(f"Syncing summaries start_block: {start}")
batch_end = start + BLOCKS_PER_SUMMARY - 1
if batch_end > end:
logger.warn("Syncing summaries is not required")
while batch_end <= end:
summary_result = ethereum_summary(db_session, start, batch_end)
push_summary(summary_result)
logger.info(f"Pushed summary of blocks : {start}-{batch_end}")
start = batch_end + 1
batch_end += BLOCKS_PER_SUMMARY
if start == end:
return end
else:
return start - 1
def ethereum_sync_handler(args: argparse.Namespace) -> None:
web3_client = web3_client_from_cli_or_env(args)
with yield_db_session_ctx() as db_session:
logger.info("Initial labeling:")
last_labeled = sync_labels(db_session, web3_client, args.start)
logger.info("Initial summary creation:")
last_summary_created = sync_summaries(
db_session,
args.start,
last_labeled,
)
while True:
print(f"Labeling blocks {start}-{end}")
add_labels(web3_client, db_session, start, end)
end_time = datetime.fromtimestamp(latest_block.timestamp, timezone.utc)
print(f"Creating summary with endtime={end_time}")
result = ethereum_summary(db_session, end_time)
push_summary(result, end, humbug_token)
sleep_time = 60 * 60
print(f"Going to sleep for:{sleep_time}s")
logger.info("Syncing")
last_labeled = sync_labels(db_session, web3_client, last_labeled + 1)
last_summary_created = sync_summaries(
db_session,
last_summary_created + 1,
last_labeled,
)
sleep_time = 10 * 60
logger.info(f"Going to sleep for {sleep_time}s")
time.sleep(sleep_time)
start = end + 1
latest_block = get_latest_block_from_db(db_session)
end = latest_block.block_number
def ethereum_label_handler(args: argparse.Namespace) -> None:
web3_client = web3_client_from_cli_or_env(args)
@ -96,43 +225,63 @@ def ethereum_label_handler(args: argparse.Namespace) -> None:
add_labels(web3_client, db_session, args.start, args.end, args.address)
def push_summary(result: Dict[str, Any], end_block_no: int, humbug_token: str):
def push_summary(result: Dict[str, Any], humbug_token: Optional[str] = None):
if humbug_token is None:
humbug_token = NFT_HUMBUG_TOKEN
title = (
f"NFT activity on the Ethereum blockchain: crawled at: {result['crawled_at'] })"
)
tags = [
f"crawler_version:{MOONCRAWL_VERSION}",
f"summary_id:{result.get(SUMMARY_KEY_ID, '')}",
f"start_block:{result.get(SUMMARY_KEY_START_BLOCK)}",
f"end_block:{result.get(SUMMARY_KEY_END_BLOCK)}",
]
# Add an "error:missing_blocks" tag for all summaries in which the number of blocks processed
# is not equal to the expected number of blocks.
args = result.get(SUMMARY_KEY_ARGS, {})
args_start = args.get("start")
args_end = args.get("end")
expected_num_blocks = None
if args_start is not None and args_end is not None:
expected_num_blocks = cast(int, args_end) - cast(int, args_start) + 1
num_blocks = result.get(SUMMARY_KEY_NUM_BLOCKS)
if (
expected_num_blocks is None
or num_blocks is None
or num_blocks != expected_num_blocks
):
tags.append("error:missing_blocks")
# TODO(yhtyyar, zomglings): Also add checkpoints in database for nft labelling. This way, we can
# add an "error:stale" tag to summaries generated before nft labels were processed for the
# block range in the summary.
created_at = result.get("date_range", {}).get("end_time")
title = f"NFT activity on the Ethereum blockchain: end time: {result['crawled_at'] } (block {end_block_no})"
publish_json(
"nft_ethereum",
humbug_token,
title,
result,
tags=[f"crawler_version:{MOONCRAWL_VERSION}", f"end_block:{end_block_no}"],
wait=False,
tags=tags,
wait=True,
created_at=created_at,
)
def ethereum_summary_handler(args: argparse.Namespace) -> None:
with yield_db_session_ctx() as db_session:
result = ethereum_summary(db_session, args.end)
# humbug_token = args.humbug
# if humbug_token is None:
# humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN")
# if humbug_token:
# title = f"NFT activity on the Ethereum blockchain: {start_time} (block {start_block}) to {end_time} (block {end_block})"
# publish_json(
# "nft_ethereum",
# humbug_token,
# title,
# result,
# tags=[f"crawler_version:{MOONCRAWL_VERSION}"],
# wait=False,
# )
with yield_db_session_ctx() as db_session:
result = ethereum_summary(db_session, args.start, args.end)
push_summary(result, args.humbug)
with args.outfile as ofp:
json.dump(result, ofp)
def main() -> None:
time_now = datetime.now(timezone.utc)
parser = argparse.ArgumentParser(description="Moonstream NFT crawlers")
parser.set_defaults(func=lambda _: parser.print_help())
subcommands = parser.add_subparsers(description="Subcommands")
@ -180,13 +329,28 @@ def main() -> None:
parser_ethereum_summary = subparsers_ethereum.add_parser(
"summary", description="Generate Ethereum NFT summary"
)
parser_ethereum_summary.add_argument(
"-s",
"--start",
type=int,
required=True,
help=f"Start block for window to calculate NFT statistics",
)
parser_ethereum_summary.add_argument(
"-e",
"--end",
type=dateutil.parser.parse,
default=time_now.isoformat(),
help=f"End time for window to calculate NFT statistics (default: {time_now.isoformat()})",
type=int,
required=True,
help=f"End block for window to calculate NFT statistics",
)
parser_ethereum_summary.add_argument(
"-o",
"--outfile",
type=argparse.FileType("w"),
default=sys.stdout,
help="Optional file to write output to. By default, prints to stdout.",
)
parser_ethereum_summary.add_argument(
"--humbug",
default=None,
@ -196,17 +360,10 @@ def main() -> None:
"MOONSTREAM_HUMBUG_TOKEN environment variable)"
),
)
parser_ethereum_summary.add_argument(
"-o",
"--outfile",
type=argparse.FileType("w"),
default=sys.stdout,
help="Optional file to write output to. By default, prints to stdout.",
)
parser_ethereum_summary.set_defaults(func=ethereum_summary_handler)
parser_ethereum_sync = subparsers_ethereum.add_parser(
"sync",
"synchronize",
description="Label addresses and transactions in databse using crawled NFT transfer information, sync mode",
)
parser_ethereum_sync.add_argument(

139
crawlers/mooncrawl/mooncrawl/nft/ethereum.py
Wyświetl plik

@ -1,5 +1,5 @@
from dataclasses import dataclass, asdict
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
import json
import logging
from hexbytes.main import HexBytes
@ -20,6 +20,8 @@ from web3 import Web3
from web3.types import FilterParams, LogReceipt
from web3._utils.events import get_event_data
from ..reporter import reporter
# Default length (in blocks) of an Ethereum NFT crawl.
DEFAULT_CRAWL_LENGTH = 100
@ -31,7 +33,11 @@ logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
# Summary keys
SUMMARY_KEY_BLOCKS = "blocks"
SUMMARY_KEY_ID = "summary_id"
SUMMARY_KEY_ARGS = "args"
SUMMARY_KEY_START_BLOCK = "start_block"
SUMMARY_KEY_END_BLOCK = "end_block"
SUMMARY_KEY_NUM_BLOCKS = "num_blocks"
SUMMARY_KEY_NUM_TRANSACTIONS = "num_transactions"
SUMMARY_KEY_TOTAL_VALUE = "total_value"
SUMMARY_KEY_NFT_TRANSFERS = "nft_transfers"
@ -41,7 +47,11 @@ SUMMARY_KEY_NFT_PURCHASERS = "nft_owners"
SUMMARY_KEY_NFT_MINTERS = "nft_minters"
SUMMARY_KEYS = [
SUMMARY_KEY_BLOCKS,
SUMMARY_KEY_ID,
SUMMARY_KEY_ARGS,
SUMMARY_KEY_START_BLOCK,
SUMMARY_KEY_END_BLOCK,
SUMMARY_KEY_NUM_BLOCKS,
SUMMARY_KEY_NUM_TRANSACTIONS,
SUMMARY_KEY_TOTAL_VALUE,
SUMMARY_KEY_NFT_TRANSFERS,
@ -52,16 +62,16 @@ SUMMARY_KEYS = [
]
# First abi is for old NFT's like crypto kitties
# The erc721 standart requieres that Transfer event is indexed for all arguments
# That is how we get distinguished from erc20 transfer events
# Second abi is for old NFT's like crypto kitties
erc721_transfer_event_abis = [
{
"anonymous": False,
"inputs": [
{"indexed": False, "name": "from", "type": "address"},
{"indexed": False, "name": "to", "type": "address"},
{"indexed": False, "name": "tokenId", "type": "uint256"},
{"indexed": True, "name": "from", "type": "address"},
{"indexed": True, "name": "to", "type": "address"},
{"indexed": True, "name": "tokenId", "type": "uint256"},
],
"name": "Transfer",
"type": "event",
@ -69,9 +79,9 @@ erc721_transfer_event_abis = [
{
"anonymous": False,
"inputs": [
{"indexed": True, "name": "from", "type": "address"},
{"indexed": True, "name": "to", "type": "address"},
{"indexed": True, "name": "tokenId", "type": "uint256"},
{"indexed": False, "name": "from", "type": "address"},
{"indexed": False, "name": "to", "type": "address"},
{"indexed": False, "name": "tokenId", "type": "uint256"},
],
"name": "Transfer",
"type": "event",
@ -129,7 +139,6 @@ class NFTContract:
address: str
name: Optional[str] = None
symbol: Optional[str] = None
total_supply: Optional[str] = None
def get_erc721_contract_info(w3: Web3, address: str) -> NFTContract:
@ -148,14 +157,10 @@ def get_erc721_contract_info(w3: Web3, address: str) -> NFTContract:
except:
logger.error(f"Could not get symbol for potential NFT contract: {address}")
totalSupply: Optional[str] = None
try:
totalSupply = contract.functions.totalSupply().call()
except:
logger.error(f"Could not get totalSupply for potential NFT contract: {address}")
return NFTContract(
address=address, name=name, symbol=symbol, total_supply=totalSupply
address=address,
name=name,
symbol=symbol,
)
@ -185,13 +190,6 @@ class NFTTransfer:
is_mint: bool = False
def get_value_by_tx(w3: Web3, tx_hash: HexBytes):
print(f"Trying to get tx: {tx_hash.hex()}")
tx = w3.eth.get_transaction(tx_hash)
print("got it")
return tx["value"]
def decode_nft_transfer_data(w3: Web3, log: LogReceipt) -> Optional[NFTTransferRaw]:
for abi in erc721_transfer_event_abis:
try:
@ -311,7 +309,6 @@ def label_erc721_addresses(
label_data={
"name": contract_info.name,
"symbol": contract_info.symbol,
"totalSupply": contract_info.total_supply,
},
)
)
@ -322,6 +319,7 @@ def label_erc721_addresses(
db_session.bulk_save_objects(labels)
db_session.commit()
except Exception as e:
reporter.error_report(e, ["nft-crawler"], True)
db_session.rollback()
logger.error(f"Failed to save labels to db:\n{e}")
@ -377,6 +375,7 @@ def label_transfers(
db_session.bulk_save_objects(new_labels)
db_session.commit()
except Exception as e:
reporter.error_report(e, ["nft-crawler"], True)
db_session.rollback()
logger.error("Could not write transfer/mint labels to database")
logger.error(e)
@ -485,20 +484,19 @@ def add_labels(
pbar.close()
def time_bounded_summary(
def block_bounded_summary(
db_session: Session,
start_time: datetime,
end_time: datetime,
start_block: int,
end_block: int,
) -> Dict[str, Any]:
"""
Produces a summary of Ethereum NFT activity between the given start_time and end_time (inclusive).
"""
start_timestamp = int(start_time.timestamp())
end_timestamp = int(end_time.timestamp())
summary_id = f"nft-ethereum-start-{start_block}-end-{end_block}"
time_filter = and_(
EthereumBlock.timestamp >= start_timestamp,
EthereumBlock.timestamp <= end_timestamp,
block_filter = and_(
EthereumBlock.block_number >= start_block,
EthereumBlock.block_number <= end_block,
)
transactions_query = (
@ -507,7 +505,7 @@ def time_bounded_summary(
EthereumBlock,
EthereumTransaction.block_number == EthereumBlock.block_number,
)
.filter(time_filter)
.filter(block_filter)
)
def nft_query(label: str) -> Query:
@ -529,7 +527,7 @@ def time_bounded_summary(
EthereumBlock,
EthereumTransaction.block_number == EthereumBlock.block_number,
)
.filter(time_filter)
.filter(block_filter)
.filter(EthereumLabel.label == label)
)
return query
@ -556,7 +554,7 @@ def time_bounded_summary(
EthereumTransaction.block_number == EthereumBlock.block_number,
)
.filter(EthereumLabel.label == label)
.filter(time_filter)
.filter(block_filter)
.order_by(
# Without "transfer_value" and "owner_address" as sort keys, the final distinct query
# does not seem to be deterministic.
@ -572,21 +570,30 @@ def time_bounded_summary(
purchaser_query = holder_query(TRANSFER_LABEL)
minter_query = holder_query(MINT_LABEL)
blocks_result: Dict[str, int] = {}
min_block = (
db_session.query(func.min(EthereumBlock.block_number))
.filter(time_filter)
.scalar()
)
max_block = (
db_session.query(func.max(EthereumBlock.block_number))
.filter(time_filter)
.scalar()
blocks = (
db_session.query(EthereumBlock)
.filter(block_filter)
.order_by(EthereumBlock.block_number.asc())
)
first_block = None
last_block = None
num_blocks = 0
for block in blocks:
if num_blocks == 0:
min_block = block
max_block = block
num_blocks += 1
start_time = None
end_time = None
if min_block is not None:
blocks_result["start"] = min_block
first_block = min_block.block_number
start_time = datetime.fromtimestamp(
min_block.timestamp, timezone.utc
).isoformat()
if max_block is not None:
blocks_result["end"] = max_block
last_block = max_block.block_number
end_time = datetime.fromtimestamp(max_block.timestamp, timezone.utc).isoformat()
num_transactions = transactions_query.distinct(EthereumTransaction.hash).count()
num_transfers = transfer_query.distinct(EthereumTransaction.hash).count()
@ -614,12 +621,16 @@ def time_bounded_summary(
result = {
"date_range": {
"start_time": start_time.isoformat(),
"start_time": start_time,
"include_start": True,
"end_time": end_time.isoformat(),
"end_time": end_time,
"include_end": True,
},
SUMMARY_KEY_BLOCKS: blocks_result,
SUMMARY_KEY_ID: summary_id,
SUMMARY_KEY_ARGS: {"start": start_block, "end": end_block},
SUMMARY_KEY_START_BLOCK: first_block,
SUMMARY_KEY_END_BLOCK: last_block,
SUMMARY_KEY_NUM_BLOCKS: num_blocks,
SUMMARY_KEY_NUM_TRANSACTIONS: f"{num_transactions}",
SUMMARY_KEY_TOTAL_VALUE: f"{total_value}",
SUMMARY_KEY_NFT_TRANSFERS: f"{num_transfers}",
@ -632,28 +643,12 @@ def time_bounded_summary(
return result
def summary(db_session: Session, end_time: datetime) -> Dict[str, Any]:
def summary(db_session: Session, start_block: int, end_block: int) -> Dict[str, Any]:
"""
Produces a summary of all Ethereum NFT activity:
1. From 1 hour before end_time to end_time
2. From 1 day before end_time to end_time
3. From 1 week before end_time to end_time
From 1 hour before end_time to end_time
"""
start_times = {
"hour": end_time - timedelta(hours=1),
"day": end_time - timedelta(days=1),
"week": end_time - timedelta(weeks=1),
}
summaries = {
period: time_bounded_summary(db_session, start_time, end_time)
for period, start_time in start_times.items()
}
def aggregate_summary(key: str) -> Dict[str, Any]:
return {period: summary.get(key) for period, summary in summaries.items()}
result: Dict[str, Any] = {
summary_key: aggregate_summary(summary_key) for summary_key in SUMMARY_KEYS
}
result["crawled_at"] = end_time.isoformat()
result = block_bounded_summary(db_session, start_block, end_block)
result["crawled_at"] = datetime.utcnow().isoformat()
return result

13
crawlers/mooncrawl/mooncrawl/publish.py
Wyświetl plik

@ -1,3 +1,4 @@
from datetime import datetime
import json
import os
from typing import Any, Dict, List, Optional
@ -12,6 +13,7 @@ def publish_json(
content: Dict[str, Any],
tags: Optional[List[str]] = None,
wait: bool = True,
created_at: Optional[str] = None,
) -> None:
spire_api_url = os.environ.get(
"MOONSTREAM_SPIRE_API_URL", "https://spire.bugout.dev"
@ -26,9 +28,18 @@ def publish_json(
headers = {
"Authorization": f"Bearer {humbug_token}",
}
request_body = {"title": title, "content": json.dumps(content), "tags": tags}
request_body = {
"title": title,
"content": json.dumps(content),
"tags": tags,
}
if created_at is not None:
request_body["created_at"] = created_at
query_parameters = {"sync": wait}
response = requests.post(
report_url, headers=headers, json=request_body, params=query_parameters
)
response.raise_for_status()

14
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -1,4 +1,5 @@
import os
from typing import cast
# Bugout
HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
@ -18,3 +19,16 @@ except:
# Etherscan
MOONSTREAM_ETHERSCAN_TOKEN = os.environ.get("MOONSTREAM_ETHERSCAN_TOKEN")
# NFT crawler
NFT_HUMBUG_TOKEN = os.environ.get("NFT_HUMBUG_TOKEN", "")
if NFT_HUMBUG_TOKEN == "":
raise ValueError("NFT_HUMBUG_TOKEN env variable is not set")
MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN", "")
if MOONSTREAM_ADMIN_ACCESS_TOKEN == "":
raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN env variable is not set")
MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID", "")
if MOONSTREAM_DATA_JOURNAL_ID == "":
raise ValueError("MOONSTREAM_DATA_JOURNAL_ID env variable is not set")

1
crawlers/mooncrawl/requirements.txt
Wyświetl plik

@ -6,6 +6,7 @@ bitarray==1.2.2
black==21.8b0
boto3==1.18.40
botocore==1.21.40
bugout==0.1.17
certifi==2021.5.30
chardet==4.0.0
charset-normalizer==2.0.4

3
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -7,3 +7,6 @@ export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket for smart contracts>"
export MOONSTREAM_HUMBUG_TOKEN="<Token for crawlers store data via Humbug>"
export COINMARKETCAP_API_KEY="<API key to parse conmarketcap>"
export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"
export MOONSTREAM_DATA_JOURNAL_ID="<Bugout journal id for moonstream>"
export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Bugout access token for moonstream>"
export NFT_HUMBUG_TOKEN="<Token for nft crawler>"

3
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -32,13 +32,14 @@ setup(
package_data={"mooncrawl": ["py.typed"]},
zip_safe=False,
install_requires=[
"boto3",
"bugout >= 0.1.17",
"moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@a4fff6498f66789934d4af26fd42a8cfb6e5eed5#egg=moonstreamdb&subdirectory=db",
"humbug",
"python-dateutil",
"requests",
"tqdm",
"web3",
"boto3",
],
extras_require={
"dev": ["black", "mypy", "types-requests", "types-python-dateutil"]