From 5b6d06cf99e6c2a4aac414c588c56c2ad8af18a8 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Wed, 15 Sep 2021 12:55:03 +0300 Subject: [PATCH 01/24] added ethereum-nft service --- crawlers/deploy/ethereum-nft.service | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 crawlers/deploy/ethereum-nft.service diff --git a/crawlers/deploy/ethereum-nft.service b/crawlers/deploy/ethereum-nft.service new file mode 100644 index 00000000..ee4273da --- /dev/null +++ b/crawlers/deploy/ethereum-nft.service @@ -0,0 +1,11 @@ +[Unit] +Description=Labels nft transactions and sends summary to humbug +After=network.target + +[Service] +Type=oneshot +User=ubuntu +Group=www-data +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/mooncrawl-secrets/app.env +ExecStart=/usr/bin/bash -c '/home/ubuntu/mooncrawl-env/bin/python -m mooncrawl.nft.cli ethereum sync' From a093946a7a6ef6dabe86737f7132d4faf811f9c2 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Thu, 16 Sep 2021 10:32:49 +0300 Subject: [PATCH 02/24] changed summary schema, now it requires start and end blocks --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 164 +++++++++++++------ crawlers/mooncrawl/mooncrawl/nft/ethereum.py | 96 ++++++----- 2 files changed, 157 insertions(+), 103 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index 7063861a..b6f980f7 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -3,13 +3,16 @@ A command line tool to crawl information about NFTs from various sources. """ import argparse from datetime import datetime, timedelta, timezone +import dateutil.parser +from dateutil.relativedelta import relativedelta import json +import logging import os import sys import time -from typing import Any, Dict, cast +from typing import Any, Dict, cast, Optional + -import dateutil.parser from moonstreamdb.db import yield_db_session_ctx from moonstreamdb.models import EthereumBlock from sqlalchemy.orm.session import Session @@ -21,6 +24,11 @@ from ..publish import publish_json from ..settings import MOONSTREAM_IPC_PATH from ..version import MOONCRAWL_VERSION +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +BLOCKS_PER_SUMMARY = 40 + def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3: """ @@ -47,35 +55,86 @@ def get_latest_block_from_db(db_session: Session): ) +def get_latest_summary_block() -> Optional[int]: + pass + + +def get_latest_nft_event_date() -> Optional[datetime]: + pass + + +def sync_labels( + db_session: Session, web3_client: Web3, start: Optional[int] +) -> EthereumBlock: + if start is None: + logger.info( + "Syncing start block is not given, getting it from latest nft label in db" + ) + start_date = get_latest_nft_event_date() + if start_date is None: + logger.warning( + "Didn't find any nft labels in db, starting sync from 3 month before now" + ) + time_now = datetime.now(timezone.utc) + start_date = time_now - relativedelta(months=3) + + start = ( + db_session.query(EthereumBlock) + .filter(EthereumBlock.timestamp >= start_date.timestamp()) + .order_by(EthereumBlock.timestamp.asc()) + .limit(1) + .one() + ).block_number + logger.info(f"Start block: {start}, date: {start_date}") + latest_block = get_latest_block_from_db(db_session) + end = latest_block.block_number + assert start <= end, f"Start block {start} is greater than latest_block {end} in db" + logger.info(f"Labeling blocks {start}-{end}") + add_labels(web3_client, db_session, start, end) + return latest_block + + +def sync_summaries(db_session: Session, end: int, start: Optional[int]): + if start is None: + logger.info( + "Syncing start time is not given, getting it from latest nft label in db" + ) + start = get_latest_summary_block() + if start is None: + time_now = datetime.now(timezone.utc) + start_date = time_now - relativedelta(months=3) + start = ( + db_session.query(EthereumBlock) + .filter(EthereumBlock.timestamp >= start_date.timestamp()) + .order_by(EthereumBlock.timestamp.asc()) + .limit(1) + .one() + ).block_number + start += 1 + + while start < end: + current_end = start + BLOCKS_PER_SUMMARY - 1 + current_end_time = ( + db_session.query(EthereumBlock) + .filter(EthereumBlock.block_number <= current_end) + .order_by(EthereumBlock.block_number.desc()) + .limit(1) + .one() + ).timestamp + summary_result = ethereum_summary( + db_session, datetime.fromtimestamp(current_end_time, timezone.utc) + ) + + def ethereum_sync_handler(args: argparse.Namespace) -> None: web3_client = web3_client_from_cli_or_env(args) + humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") if humbug_token is None: raise ValueError("MOONSTREAM_HUMBUG_TOKEN env variable is not set") with yield_db_session_ctx() as db_session: - start = args.start - if start is None: - time_now = datetime.now(timezone.utc) - week_ago = time_now - timedelta(weeks=1) - start = ( - db_session.query(EthereumBlock) - .filter(EthereumBlock.timestamp >= week_ago.timestamp()) - .order_by(EthereumBlock.timestamp.asc()) - .limit(1) - .one() - ).block_number - - latest_block = get_latest_block_from_db(db_session) - end = latest_block.block_number - assert ( - start <= end - ), f"Start block {start} is greater than latest_block {end} in db" - while True: - print(f"Labeling blocks {start}-{end}") - add_labels(web3_client, db_session, start, end) - end_time = datetime.fromtimestamp(latest_block.timestamp, timezone.utc) print(f"Creating summary with endtime={end_time}") result = ethereum_summary(db_session, end_time) @@ -96,36 +155,30 @@ def ethereum_label_handler(args: argparse.Namespace) -> None: add_labels(web3_client, db_session, args.start, args.end, args.address) -def push_summary(result: Dict[str, Any], end_block_no: int, humbug_token: str): +def push_summary(result: Dict[str, Any], humbug_token: str): - title = f"NFT activity on the Ethereum blockchain: end time: {result['crawled_at'] } (block {end_block_no})" + title = ( + f"NFT activity on the Ethereum blockchain: end time: {result['crawled_at'] })" + ) publish_json( "nft_ethereum", humbug_token, title, result, - tags=[f"crawler_version:{MOONCRAWL_VERSION}", f"end_block:{end_block_no}"], + tags=[f"crawler_version:{MOONCRAWL_VERSION}"], wait=False, ) def ethereum_summary_handler(args: argparse.Namespace) -> None: - with yield_db_session_ctx() as db_session: - result = ethereum_summary(db_session, args.end) - # humbug_token = args.humbug - # if humbug_token is None: - # humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") - # if humbug_token: - # title = f"NFT activity on the Ethereum blockchain: {start_time} (block {start_block}) to {end_time} (block {end_block})" - # publish_json( - # "nft_ethereum", - # humbug_token, - # title, - # result, - # tags=[f"crawler_version:{MOONCRAWL_VERSION}"], - # wait=False, - # ) + with yield_db_session_ctx() as db_session: + result = ethereum_summary(db_session, args.start, args.end) + humbug_token = args.humbug + if humbug_token is None: + humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") + if humbug_token: + push_summary(result, humbug_token) with args.outfile as ofp: json.dump(result, ofp) @@ -181,20 +234,18 @@ def main() -> None: "summary", description="Generate Ethereum NFT summary" ) parser_ethereum_summary.add_argument( - "-e", - "--end", - type=dateutil.parser.parse, - default=time_now.isoformat(), - help=f"End time for window to calculate NFT statistics (default: {time_now.isoformat()})", + "-s", + "--start", + type=int, + required=True, + help=f"Start block for window to calculate NFT statistics", ) parser_ethereum_summary.add_argument( - "--humbug", - default=None, - help=( - "If you would like to write this data to a Moonstream journal, please provide a Humbug " - "token for that here. (This argument overrides any value set in the " - "MOONSTREAM_HUMBUG_TOKEN environment variable)" - ), + "-e", + "--end", + type=int, + required=True, + help=f"End block for window to calculate NFT statistics", ) parser_ethereum_summary.add_argument( "-o", @@ -203,6 +254,13 @@ def main() -> None: default=sys.stdout, help="Optional file to write output to. By default, prints to stdout.", ) + + parser_ethereum_summary.add_argument( + "-humbug", + "--humbug", + default=None, + help="Humbug token, if given summary will send there", + ) parser_ethereum_summary.set_defaults(func=ethereum_summary_handler) parser_ethereum_sync = subparsers_ethereum.add_parser( diff --git a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py index 9f84a6ee..d695f31a 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, asdict -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import json import logging from hexbytes.main import HexBytes @@ -31,7 +31,9 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) # Summary keys -SUMMARY_KEY_BLOCKS = "blocks" +SUMMARY_KEY_START_BLOCK = "start_block" +SUMMARY_KEY_END_BLOCK = "end_block" +SUMMARY_KEY_NUM_BLOCKS = "num_blocks" SUMMARY_KEY_NUM_TRANSACTIONS = "num_transactions" SUMMARY_KEY_TOTAL_VALUE = "total_value" SUMMARY_KEY_NFT_TRANSFERS = "nft_transfers" @@ -41,7 +43,9 @@ SUMMARY_KEY_NFT_PURCHASERS = "nft_owners" SUMMARY_KEY_NFT_MINTERS = "nft_minters" SUMMARY_KEYS = [ - SUMMARY_KEY_BLOCKS, + SUMMARY_KEY_START_BLOCK, + SUMMARY_KEY_END_BLOCK, + SUMMARY_KEY_NUM_BLOCKS, SUMMARY_KEY_NUM_TRANSACTIONS, SUMMARY_KEY_TOTAL_VALUE, SUMMARY_KEY_NFT_TRANSFERS, @@ -485,20 +489,17 @@ def add_labels( pbar.close() -def time_bounded_summary( +def block_bounded_summary( db_session: Session, - start_time: datetime, - end_time: datetime, + start_block: int, + end_block: int, ) -> Dict[str, Any]: """ Produces a summary of Ethereum NFT activity between the given start_time and end_time (inclusive). """ - start_timestamp = int(start_time.timestamp()) - end_timestamp = int(end_time.timestamp()) - - time_filter = and_( - EthereumBlock.timestamp >= start_timestamp, - EthereumBlock.timestamp <= end_timestamp, + block_filter = and_( + EthereumBlock.block_number >= start_block, + EthereumBlock.block_number <= end_block, ) transactions_query = ( @@ -507,7 +508,7 @@ def time_bounded_summary( EthereumBlock, EthereumTransaction.block_number == EthereumBlock.block_number, ) - .filter(time_filter) + .filter(block_filter) ) def nft_query(label: str) -> Query: @@ -529,7 +530,7 @@ def time_bounded_summary( EthereumBlock, EthereumTransaction.block_number == EthereumBlock.block_number, ) - .filter(time_filter) + .filter(block_filter) .filter(EthereumLabel.label == label) ) return query @@ -556,7 +557,7 @@ def time_bounded_summary( EthereumTransaction.block_number == EthereumBlock.block_number, ) .filter(EthereumLabel.label == label) - .filter(time_filter) + .filter(block_filter) .order_by( # Without "transfer_value" and "owner_address" as sort keys, the final distinct query # does not seem to be deterministic. @@ -572,21 +573,30 @@ def time_bounded_summary( purchaser_query = holder_query(TRANSFER_LABEL) minter_query = holder_query(MINT_LABEL) - blocks_result: Dict[str, int] = {} - min_block = ( - db_session.query(func.min(EthereumBlock.block_number)) - .filter(time_filter) - .scalar() - ) - max_block = ( - db_session.query(func.max(EthereumBlock.block_number)) - .filter(time_filter) - .scalar() + blocks = ( + db_session.query(EthereumBlock) + .filter(block_filter) + .order_by(EthereumBlock.block_number.asc()) ) + first_block = None + last_block = None + num_blocks = 0 + for block in blocks: + if num_blocks == 0: + min_block = block + max_block = block + num_blocks += 1 + + start_time = None + end_time = None if min_block is not None: - blocks_result["start"] = min_block + first_block = min_block.block_number + start_time = datetime.fromtimestamp( + min_block.timestamp, timezone.utc + ).isoformat() if max_block is not None: - blocks_result["end"] = max_block + last_block = max_block.block_number + end_time = datetime.fromtimestamp(max_block.timestamp, timezone.utc).isoformat() num_transactions = transactions_query.distinct(EthereumTransaction.hash).count() num_transfers = transfer_query.distinct(EthereumTransaction.hash).count() @@ -614,12 +624,14 @@ def time_bounded_summary( result = { "date_range": { - "start_time": start_time.isoformat(), + "start_time": start_time, "include_start": True, - "end_time": end_time.isoformat(), + "end_time": end_time, "include_end": True, }, - SUMMARY_KEY_BLOCKS: blocks_result, + SUMMARY_KEY_START_BLOCK: first_block, + SUMMARY_KEY_END_BLOCK: last_block, + SUMMARY_KEY_NUM_BLOCKS: num_blocks, SUMMARY_KEY_NUM_TRANSACTIONS: f"{num_transactions}", SUMMARY_KEY_TOTAL_VALUE: f"{total_value}", SUMMARY_KEY_NFT_TRANSFERS: f"{num_transfers}", @@ -632,28 +644,12 @@ def time_bounded_summary( return result -def summary(db_session: Session, end_time: datetime) -> Dict[str, Any]: +def summary(db_session: Session, start_block: int, end_block: int) -> Dict[str, Any]: """ Produces a summary of all Ethereum NFT activity: - 1. From 1 hour before end_time to end_time - 2. From 1 day before end_time to end_time - 3. From 1 week before end_time to end_time + From 1 hour before end_time to end_time """ - start_times = { - "hour": end_time - timedelta(hours=1), - "day": end_time - timedelta(days=1), - "week": end_time - timedelta(weeks=1), - } - summaries = { - period: time_bounded_summary(db_session, start_time, end_time) - for period, start_time in start_times.items() - } - def aggregate_summary(key: str) -> Dict[str, Any]: - return {period: summary.get(key) for period, summary in summaries.items()} - - result: Dict[str, Any] = { - summary_key: aggregate_summary(summary_key) for summary_key in SUMMARY_KEYS - } - result["crawled_at"] = end_time.isoformat() + result = block_bounded_summary(db_session, start_block, end_block) + result["crawled_at"] = datetime.utcnow().isoformat() return result From 33fa888117f4354a41ce1337597f32226a2423b8 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Thu, 16 Sep 2021 00:40:19 -0700 Subject: [PATCH 03/24] Small updates on CLI Improved help for --humbug option for nft summaries and removed `-humbug` form for that argument. --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index b6f980f7..655798f3 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -256,15 +256,18 @@ def main() -> None: ) parser_ethereum_summary.add_argument( - "-humbug", "--humbug", default=None, - help="Humbug token, if given summary will send there", + help=( + "If you would like to write this data to a Moonstream journal, please provide a Humbug " + "token for that here. (This argument overrides any value set in the " + "MOONSTREAM_HUMBUG_TOKEN environment variable)" + ), ) parser_ethereum_summary.set_defaults(func=ethereum_summary_handler) parser_ethereum_sync = subparsers_ethereum.add_parser( - "sync", + "synchronize", description="Label addresses and transactions in databse using crawled NFT transfer information, sync mode", ) parser_ethereum_sync.add_argument( From c134d4f6ac4989dc25b06fd090ed5e04f22f53fd Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Thu, 16 Sep 2021 02:14:22 -0700 Subject: [PATCH 04/24] Added support for created_at in Humbug reports `nft ethereum summary` crawler reports the `"date_range.end_time"` as its `created_at`. This allows us to query the Humbug journal for all summaries involving end blocks that were mined during a given stream boundary. --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 47 +++++++++++++++++--- crawlers/mooncrawl/mooncrawl/nft/ethereum.py | 8 ++++ crawlers/mooncrawl/mooncrawl/publish.py | 13 +++++- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index 655798f3..efc51b00 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -2,15 +2,14 @@ A command line tool to crawl information about NFTs from various sources. """ import argparse -from datetime import datetime, timedelta, timezone -import dateutil.parser +from datetime import datetime, timezone from dateutil.relativedelta import relativedelta import json import logging import os import sys import time -from typing import Any, Dict, cast, Optional +from typing import Any, cast, Dict, Optional from moonstreamdb.db import yield_db_session_ctx @@ -19,7 +18,13 @@ from sqlalchemy.orm.session import Session from web3 import Web3 from ..ethereum import connect -from .ethereum import summary as ethereum_summary, add_labels +from .ethereum import ( + summary as ethereum_summary, + add_labels, + SUMMARY_KEY_ARGS, + SUMMARY_KEY_ID, + SUMMARY_KEY_NUM_BLOCKS, +) from ..publish import publish_json from ..settings import MOONSTREAM_IPC_PATH from ..version import MOONCRAWL_VERSION @@ -156,17 +161,45 @@ def ethereum_label_handler(args: argparse.Namespace) -> None: def push_summary(result: Dict[str, Any], humbug_token: str): - title = ( f"NFT activity on the Ethereum blockchain: end time: {result['crawled_at'] })" ) + + tags = [ + f"crawler_version:{MOONCRAWL_VERSION}", + f"summary_id:{result.get(SUMMARY_KEY_ID, '')}", + ] + + # Add an "error:missing_blocks" tag for all summaries in which the number of blocks processed + # is not equal to the expected number of blocks. + args = result.get(SUMMARY_KEY_ARGS, {}) + args_start = args.get("start") + args_end = args.get("end") + expected_num_blocks = None + if args_start is not None and args_end is not None: + expected_num_blocks = cast(int, args_end) - cast(int, args_start) + 1 + num_blocks = result.get(SUMMARY_KEY_NUM_BLOCKS) + if ( + expected_num_blocks is None + or num_blocks is None + or num_blocks != expected_num_blocks + ): + tags.append("error:missing_blocks") + + # TODO(yhtyyar, zomglings): Also add checkpoints in database for nft labelling. This way, we can + # add an "error:stale" tag to summaries generated before nft labels were processed for the + # block range in the summary. + + created_at = result.get("date_range", {}).get("end_time") + publish_json( "nft_ethereum", humbug_token, title, result, - tags=[f"crawler_version:{MOONCRAWL_VERSION}"], - wait=False, + tags=tags, + wait=True, + created_at=created_at, ) diff --git a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py index d695f31a..c64f425f 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py @@ -31,6 +31,8 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) # Summary keys +SUMMARY_KEY_ID = "summary_id" +SUMMARY_KEY_ARGS = "args" SUMMARY_KEY_START_BLOCK = "start_block" SUMMARY_KEY_END_BLOCK = "end_block" SUMMARY_KEY_NUM_BLOCKS = "num_blocks" @@ -43,6 +45,8 @@ SUMMARY_KEY_NFT_PURCHASERS = "nft_owners" SUMMARY_KEY_NFT_MINTERS = "nft_minters" SUMMARY_KEYS = [ + SUMMARY_KEY_ID, + SUMMARY_KEY_ARGS, SUMMARY_KEY_START_BLOCK, SUMMARY_KEY_END_BLOCK, SUMMARY_KEY_NUM_BLOCKS, @@ -497,6 +501,8 @@ def block_bounded_summary( """ Produces a summary of Ethereum NFT activity between the given start_time and end_time (inclusive). """ + summary_id = f"nft-ethereum-start-{start_block}-end-{end_block}" + block_filter = and_( EthereumBlock.block_number >= start_block, EthereumBlock.block_number <= end_block, @@ -629,6 +635,8 @@ def block_bounded_summary( "end_time": end_time, "include_end": True, }, + SUMMARY_KEY_ID: summary_id, + SUMMARY_KEY_ARGS: {"start": start_block, "end": end_block}, SUMMARY_KEY_START_BLOCK: first_block, SUMMARY_KEY_END_BLOCK: last_block, SUMMARY_KEY_NUM_BLOCKS: num_blocks, diff --git a/crawlers/mooncrawl/mooncrawl/publish.py b/crawlers/mooncrawl/mooncrawl/publish.py index bc8a73ae..16a4ed2b 100644 --- a/crawlers/mooncrawl/mooncrawl/publish.py +++ b/crawlers/mooncrawl/mooncrawl/publish.py @@ -1,3 +1,4 @@ +from datetime import datetime import json import os from typing import Any, Dict, List, Optional @@ -12,6 +13,7 @@ def publish_json( content: Dict[str, Any], tags: Optional[List[str]] = None, wait: bool = True, + created_at: Optional[str] = None, ) -> None: spire_api_url = os.environ.get( "MOONSTREAM_SPIRE_API_URL", "https://spire.bugout.dev" @@ -26,9 +28,18 @@ def publish_json( headers = { "Authorization": f"Bearer {humbug_token}", } - request_body = {"title": title, "content": json.dumps(content), "tags": tags} + request_body = { + "title": title, + "content": json.dumps(content), + "tags": tags, + } + if created_at is not None: + request_body["created_at"] = created_at + query_parameters = {"sync": wait} + response = requests.post( report_url, headers=headers, json=request_body, params=query_parameters ) + response.raise_for_status() From 42f6d893ccfb1192ca65f53eb9ad4734f61ebe30 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Thu, 16 Sep 2021 18:52:07 +0300 Subject: [PATCH 05/24] added sync functionality --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 157 +++++++++++++++--------- 1 file changed, 99 insertions(+), 58 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index efc51b00..90b96818 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -13,7 +13,7 @@ from typing import Any, cast, Dict, Optional from moonstreamdb.db import yield_db_session_ctx -from moonstreamdb.models import EthereumBlock +from moonstreamdb.models import EthereumBlock, EthereumTransaction, EthereumLabel from sqlalchemy.orm.session import Session from web3 import Web3 @@ -21,6 +21,8 @@ from ..ethereum import connect from .ethereum import ( summary as ethereum_summary, add_labels, + MINT_LABEL, + TRANSFER_LABEL, SUMMARY_KEY_ARGS, SUMMARY_KEY_ID, SUMMARY_KEY_NUM_BLOCKS, @@ -61,53 +63,45 @@ def get_latest_block_from_db(db_session: Session): def get_latest_summary_block() -> Optional[int]: - pass + return None -def get_latest_nft_event_date() -> Optional[datetime]: - pass +def get_latest_nft_labeled_block(db_session: Session) -> Optional[int]: + + query = ( + db_session.query( + EthereumLabel.label, + EthereumTransaction.hash, + EthereumBlock.block_number, + ) + .join( + EthereumTransaction, + EthereumLabel.transaction_hash == EthereumTransaction.hash, + ) + .join( + EthereumBlock, + EthereumTransaction.block_number == EthereumBlock.block_number, + ) + .filter(EthereumLabel.label.in_([MINT_LABEL, TRANSFER_LABEL])) + .order_by(EthereumBlock.block_number.desc()) + .limit(1) + ) + + return query.one_or_none().block_number -def sync_labels( - db_session: Session, web3_client: Web3, start: Optional[int] -) -> EthereumBlock: +def sync_labels(db_session: Session, web3_client: Web3, start: Optional[int]) -> int: if start is None: logger.info( "Syncing start block is not given, getting it from latest nft label in db" ) - start_date = get_latest_nft_event_date() - if start_date is None: + start = get_latest_nft_labeled_block(db_session) + if start is None: logger.warning( "Didn't find any nft labels in db, starting sync from 3 month before now" ) time_now = datetime.now(timezone.utc) start_date = time_now - relativedelta(months=3) - - start = ( - db_session.query(EthereumBlock) - .filter(EthereumBlock.timestamp >= start_date.timestamp()) - .order_by(EthereumBlock.timestamp.asc()) - .limit(1) - .one() - ).block_number - logger.info(f"Start block: {start}, date: {start_date}") - latest_block = get_latest_block_from_db(db_session) - end = latest_block.block_number - assert start <= end, f"Start block {start} is greater than latest_block {end} in db" - logger.info(f"Labeling blocks {start}-{end}") - add_labels(web3_client, db_session, start, end) - return latest_block - - -def sync_summaries(db_session: Session, end: int, start: Optional[int]): - if start is None: - logger.info( - "Syncing start time is not given, getting it from latest nft label in db" - ) - start = get_latest_summary_block() - if start is None: - time_now = datetime.now(timezone.utc) - start_date = time_now - relativedelta(months=3) start = ( db_session.query(EthereumBlock) .filter(EthereumBlock.timestamp >= start_date.timestamp()) @@ -115,20 +109,60 @@ def sync_summaries(db_session: Session, end: int, start: Optional[int]): .limit(1) .one() ).block_number - start += 1 + logger.info(f"Syncing labels, start block: {start}") + latest_block = get_latest_block_from_db(db_session) + end = latest_block.block_number + if start > end: + logger.warn(f"Start block {start} is greater than latest_block {end} in db") + logger.warn("Maybe ethcrawler is not syncing or nft sync is up to date") + return start - 1 + logger.info(f"Labeling blocks {start}-{end}") + add_labels(web3_client, db_session, start, end) + return latest_block.block_number - while start < end: - current_end = start + BLOCKS_PER_SUMMARY - 1 - current_end_time = ( - db_session.query(EthereumBlock) - .filter(EthereumBlock.block_number <= current_end) - .order_by(EthereumBlock.block_number.desc()) - .limit(1) - .one() - ).timestamp - summary_result = ethereum_summary( - db_session, datetime.fromtimestamp(current_end_time, timezone.utc) + +def sync_summaries( + db_session: Session, + start: Optional[int], + end: int, + humbug_token: str, +) -> int: + if start is None: + logger.info( + "Syncing start time is not given, getting it from latest nft label in db" ) + start = get_latest_summary_block() + if start is not None: + start += 1 + else: + logger.info( + "There is no entry in humbug, starting to create summaries from 3 month ago" + ) + time_now = datetime.now(timezone.utc) + start_date = time_now - relativedelta(months=3) + start = ( + db_session.query(EthereumBlock) + .filter(EthereumBlock.timestamp >= start_date.timestamp()) + .order_by(EthereumBlock.timestamp.asc()) + .limit(1) + .one() + ).block_number + + logger.info(f"Syncing summaries start_block: {start}") + batch_end = start + BLOCKS_PER_SUMMARY - 1 + if batch_end > end: + logger.warn("Syncing summaries is not required") + while batch_end <= end: + summary_result = ethereum_summary(db_session, start, batch_end) + push_summary(summary_result, humbug_token) + logger.info(f"Pushed summary of blocks : {start}-{batch_end}") + start = batch_end + 1 + batch_end += BLOCKS_PER_SUMMARY + + if start == end: + return end + else: + return start - 1 def ethereum_sync_handler(args: argparse.Namespace) -> None: @@ -139,20 +173,22 @@ def ethereum_sync_handler(args: argparse.Namespace) -> None: raise ValueError("MOONSTREAM_HUMBUG_TOKEN env variable is not set") with yield_db_session_ctx() as db_session: + logger.info("Initial labeling:") + last_labeled = sync_labels(db_session, web3_client, args.start) + logger.info("Initial summary creation:") + last_summary_created = sync_summaries( + db_session, args.start, last_labeled, humbug_token + ) while True: - end_time = datetime.fromtimestamp(latest_block.timestamp, timezone.utc) - print(f"Creating summary with endtime={end_time}") - result = ethereum_summary(db_session, end_time) - push_summary(result, end, humbug_token) - - sleep_time = 60 * 60 - print(f"Going to sleep for:{sleep_time}s") + logger.info("Syncing") + last_labeled = sync_labels(db_session, web3_client, last_labeled + 1) + last_summary_created = sync_summaries( + db_session, last_summary_created + 1, last_labeled, humbug_token + ) + sleep_time = 20 * 60 + logger.info(f"Going to sleep for {sleep_time}s") time.sleep(sleep_time) - start = end + 1 - latest_block = get_latest_block_from_db(db_session) - end = latest_block.block_number - def ethereum_label_handler(args: argparse.Namespace) -> None: web3_client = web3_client_from_cli_or_env(args) @@ -310,6 +346,11 @@ def main() -> None: required=False, help="Starting block number (inclusive if block available)", ) + parser_ethereum_sync.add_argument( + "--humbug", + default=None, + help=("Humbug token to publish summary reports"), + ) parser_ethereum_sync.set_defaults(func=ethereum_sync_handler) args = parser.parse_args() From d4d1f8b6e024cdd72653cfa5389a28f61c6a6569 Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Thu, 16 Sep 2021 09:56:12 -0700 Subject: [PATCH 06/24] [WIP] Adding /nft endpoint to the API --- backend/moonstream/api.py | 11 +++--- backend/moonstream/providers/bugout.py | 18 ++++++++++ backend/moonstream/routes/nft.py | 48 ++++++++++++++++++++++++++ backend/moonstream/routes/txinfo.py | 2 +- 4 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 backend/moonstream/routes/nft.py diff --git a/backend/moonstream/api.py b/backend/moonstream/api.py index 48ab8cda..6e687eb9 100644 --- a/backend/moonstream/api.py +++ b/backend/moonstream/api.py @@ -8,12 +8,12 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from . import data -from .routes.subscriptions import app as subscriptions_api -from .routes.users import app as users_api -from .routes.txinfo import app as txinfo_api -from .routes.streams import app as streams_api from .routes.address_info import app as addressinfo_api - +from .routes.nft import app as nft_api +from .routes.subscriptions import app as subscriptions_api +from .routes.streams import app as streams_api +from .routes.txinfo import app as txinfo_api +from .routes.users import app as users_api from .settings import ORIGINS from .version import MOONSTREAM_VERSION @@ -51,3 +51,4 @@ app.mount("/users", users_api) app.mount("/streams", streams_api) app.mount("/txinfo", txinfo_api) app.mount("/address_info", addressinfo_api) +app.mount("/nft", nft_api) diff --git a/backend/moonstream/providers/bugout.py b/backend/moonstream/providers/bugout.py index c1b08497..4a3d280b 100644 --- a/backend/moonstream/providers/bugout.py +++ b/backend/moonstream/providers/bugout.py @@ -350,3 +350,21 @@ ethereum_txpool_provider = EthereumTXPoolProvider( estimated_events_per_time_interval=50, tags=["client:ethereum-txpool-crawler-0"], ) + +nft_summary_description = """Event provider for NFT market summaries. + +This provider periodically generates NFT market summaries for the last hour of market activity. + +Currently, it summarizes the activities on the following NFT markets: +1. The Ethereum market + +This provider is currently not accessible for subscription. The data from this provider is publicly +available at the /nft endpoint.""" +nft_summary_provider = BugoutEventProvider( + event_type="nft_summary", + description=nft_summary_description, + # 40 blocks per summary, 15 seconds per block + 2 seconds wiggle room. + default_time_interval_seconds=40 * 17, + estimated_events_per_time_interval=1, + tags=["crawl_type:nft_ethereum"], +) diff --git a/backend/moonstream/routes/nft.py b/backend/moonstream/routes/nft.py new file mode 100644 index 00000000..d63b5e14 --- /dev/null +++ b/backend/moonstream/routes/nft.py @@ -0,0 +1,48 @@ +""" +Moonstream's /nft endpoints. + +These endpoints provide public access to NFT market summaries. No authentication required. +""" +import logging +from typing import Optional + +from fastapi import FastAPI, Query +from fastapi.middleware.cors import CORSMiddleware + +from . import data +from ..settings import DOCS_TARGET_PATH, ORIGINS +from ..version import MOONSTREAM_VERSION + +logger = logging.getLogger(__name__) + +tags_metadata = [ + {"name": "nft", "description": "NFT market summaries"}, +] + +app = FastAPI( + title=f"Moonstream /nft API", + description="User, token and password handlers.", + version=MOONSTREAM_VERSION, + openapi_tags=tags_metadata, + openapi_url="/openapi.json", + docs_url=None, + redoc_url=f"/{DOCS_TARGET_PATH}", +) + +app.add_middleware( + CORSMiddleware, + allow_origins=ORIGINS, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/", tags=["streams"], response_model=data.GetEventsResponse) +async def stream_handler( + start_time: int = Query(0), + end_time: Optional[int] = Query(None), + include_start: bool = Query(False), + include_end: bool = Query(False), +) -> data.GetEventsResponse: + pass diff --git a/backend/moonstream/routes/txinfo.py b/backend/moonstream/routes/txinfo.py index 8ab9a5ec..b8ddd249 100644 --- a/backend/moonstream/routes/txinfo.py +++ b/backend/moonstream/routes/txinfo.py @@ -28,7 +28,7 @@ tags_metadata = [ ] app = FastAPI( - title=f"Moonstream users API.", + title=f"Moonstream /txinfo API.", description="User, token and password handlers.", version=MOONSTREAM_VERSION, openapi_tags=tags_metadata, From 2650e84dd02d5c483cdd97af066a956ea759ef6b Mon Sep 17 00:00:00 2001 From: Neeraj Kashyap Date: Thu, 16 Sep 2021 10:23:31 -0700 Subject: [PATCH 07/24] [WIP] Getting things working with bugout nft provider --- backend/moonstream/routes/nft.py | 43 +++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/backend/moonstream/routes/nft.py b/backend/moonstream/routes/nft.py index d63b5e14..e04acab9 100644 --- a/backend/moonstream/routes/nft.py +++ b/backend/moonstream/routes/nft.py @@ -3,14 +3,25 @@ Moonstream's /nft endpoints. These endpoints provide public access to NFT market summaries. No authentication required. """ +from datetime import datetime import logging from typing import Optional -from fastapi import FastAPI, Query +from fastapi import Depends, FastAPI, Query +from moonstreamdb import db from fastapi.middleware.cors import CORSMiddleware +from sqlalchemy.orm import Session -from . import data -from ..settings import DOCS_TARGET_PATH, ORIGINS +from .. import data +from ..providers.bugout import nft_summary_provider +from ..settings import ( + bugout_client, + DOCS_TARGET_PATH, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_DATA_JOURNAL_ID, + ORIGINS, +) +from ..stream_queries import StreamQuery from ..version import MOONSTREAM_VERSION logger = logging.getLogger(__name__) @@ -44,5 +55,29 @@ async def stream_handler( end_time: Optional[int] = Query(None), include_start: bool = Query(False), include_end: bool = Query(False), + db_session: Session = Depends(db.yield_db_session), ) -> data.GetEventsResponse: - pass + stream_boundary = data.StreamBoundary( + start_time=start_time, + end_time=end_time, + include_start=include_start, + include_end=include_end, + ) + + result = nft_summary_provider.get_events( + db_session=db_session, + bugout_client=bugout_client, + data_journal_id=MOONSTREAM_DATA_JOURNAL_ID, + data_access_token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + stream_boundary=stream_boundary, + user_subscriptions={}, + query=StreamQuery(subscription_types=[nft_summary_provider.event_type]), + ) + + if result is None: + return data.GetEventsResponse(stream_boundary=stream_boundary, events=[]) + + provider_stream_boundary, events = result + return data.GetEventsResponse( + stream_boundary=provider_stream_boundary, events=events + ) From a67941439a92210cd51c10461959d679f236c4cf Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 16 Sep 2021 21:45:03 +0300 Subject: [PATCH 08/24] Small fix. --- backend/moonstream/routes/nft.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/moonstream/routes/nft.py b/backend/moonstream/routes/nft.py index e04acab9..5bc849ed 100644 --- a/backend/moonstream/routes/nft.py +++ b/backend/moonstream/routes/nft.py @@ -70,7 +70,9 @@ async def stream_handler( data_journal_id=MOONSTREAM_DATA_JOURNAL_ID, data_access_token=MOONSTREAM_ADMIN_ACCESS_TOKEN, stream_boundary=stream_boundary, - user_subscriptions={}, + user_subscriptions={ + nft_summary_provider.event_type: [nft_summary_provider.event_type] + }, query=StreamQuery(subscription_types=[nft_summary_provider.event_type]), ) From f3e9559da93113cf387ac3e3c986b212ef005266 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 21 Sep 2021 18:14:39 +0300 Subject: [PATCH 09/24] added bugout dependency --- crawlers/mooncrawl/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/crawlers/mooncrawl/requirements.txt b/crawlers/mooncrawl/requirements.txt index 4e1b23e4..79230244 100644 --- a/crawlers/mooncrawl/requirements.txt +++ b/crawlers/mooncrawl/requirements.txt @@ -6,6 +6,7 @@ bitarray==1.2.2 black==21.8b0 boto3==1.18.40 botocore==1.21.40 +bugout==0.1.17 certifi==2021.5.30 chardet==4.0.0 charset-normalizer==2.0.4 From 79287e1fd2700617d2b7a6dbf5df2bd6c75efae3 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 21 Sep 2021 18:15:36 +0300 Subject: [PATCH 10/24] fixed sync -> synchronize --- crawlers/deploy/ethereum-nft.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/deploy/ethereum-nft.service b/crawlers/deploy/ethereum-nft.service index ee4273da..abd0b1bc 100644 --- a/crawlers/deploy/ethereum-nft.service +++ b/crawlers/deploy/ethereum-nft.service @@ -8,4 +8,4 @@ User=ubuntu Group=www-data WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/mooncrawl-secrets/app.env -ExecStart=/usr/bin/bash -c '/home/ubuntu/mooncrawl-env/bin/python -m mooncrawl.nft.cli ethereum sync' +ExecStart=/usr/bin/bash -c '/home/ubuntu/mooncrawl-env/bin/python -m mooncrawl.nft.cli ethereum synchronize' From e5f9a6c1940f8c97d52a228ad6d128465def5b25 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 21 Sep 2021 18:17:24 +0300 Subject: [PATCH 11/24] removed totalSupply --- crawlers/mooncrawl/mooncrawl/nft/ethereum.py | 35 ++++++-------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py index c64f425f..c42d0ea4 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py @@ -60,16 +60,16 @@ SUMMARY_KEYS = [ ] -# First abi is for old NFT's like crypto kitties # The erc721 standart requieres that Transfer event is indexed for all arguments # That is how we get distinguished from erc20 transfer events +# Second abi is for old NFT's like crypto kitties erc721_transfer_event_abis = [ { "anonymous": False, "inputs": [ - {"indexed": False, "name": "from", "type": "address"}, - {"indexed": False, "name": "to", "type": "address"}, - {"indexed": False, "name": "tokenId", "type": "uint256"}, + {"indexed": True, "name": "from", "type": "address"}, + {"indexed": True, "name": "to", "type": "address"}, + {"indexed": True, "name": "tokenId", "type": "uint256"}, ], "name": "Transfer", "type": "event", @@ -77,9 +77,9 @@ erc721_transfer_event_abis = [ { "anonymous": False, "inputs": [ - {"indexed": True, "name": "from", "type": "address"}, - {"indexed": True, "name": "to", "type": "address"}, - {"indexed": True, "name": "tokenId", "type": "uint256"}, + {"indexed": False, "name": "from", "type": "address"}, + {"indexed": False, "name": "to", "type": "address"}, + {"indexed": False, "name": "tokenId", "type": "uint256"}, ], "name": "Transfer", "type": "event", @@ -137,7 +137,6 @@ class NFTContract: address: str name: Optional[str] = None symbol: Optional[str] = None - total_supply: Optional[str] = None def get_erc721_contract_info(w3: Web3, address: str) -> NFTContract: @@ -156,14 +155,10 @@ def get_erc721_contract_info(w3: Web3, address: str) -> NFTContract: except: logger.error(f"Could not get symbol for potential NFT contract: {address}") - totalSupply: Optional[str] = None - try: - totalSupply = contract.functions.totalSupply().call() - except: - logger.error(f"Could not get totalSupply for potential NFT contract: {address}") - return NFTContract( - address=address, name=name, symbol=symbol, total_supply=totalSupply + address=address, + name=name, + symbol=symbol, ) @@ -193,13 +188,6 @@ class NFTTransfer: is_mint: bool = False -def get_value_by_tx(w3: Web3, tx_hash: HexBytes): - print(f"Trying to get tx: {tx_hash.hex()}") - tx = w3.eth.get_transaction(tx_hash) - print("got it") - return tx["value"] - - def decode_nft_transfer_data(w3: Web3, log: LogReceipt) -> Optional[NFTTransferRaw]: for abi in erc721_transfer_event_abis: try: @@ -319,7 +307,6 @@ def label_erc721_addresses( label_data={ "name": contract_info.name, "symbol": contract_info.symbol, - "totalSupply": contract_info.total_supply, }, ) ) @@ -396,7 +383,7 @@ def add_labels( from_block: Optional[int] = None, to_block: Optional[int] = None, contract_address: Optional[str] = None, - batch_size: int = 100, + batch_size: int = 50, ) -> None: """ Crawls blocks between from_block and to_block checking for NFT mints and transfers. From b173b617ee5fd91909fb6a504ae81ce071b35880 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 21 Sep 2021 18:17:49 +0300 Subject: [PATCH 12/24] made synchronize work properly --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 69 +++++++++++++++++++++---- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index 90b96818..cf50407e 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -11,7 +11,8 @@ import sys import time from typing import Any, cast, Dict, Optional - +from bugout.app import Bugout +from bugout.journal import SearchOrder from moonstreamdb.db import yield_db_session_ctx from moonstreamdb.models import EthereumBlock, EthereumTransaction, EthereumLabel from sqlalchemy.orm.session import Session @@ -26,6 +27,8 @@ from .ethereum import ( SUMMARY_KEY_ARGS, SUMMARY_KEY_ID, SUMMARY_KEY_NUM_BLOCKS, + SUMMARY_KEY_START_BLOCK, + SUMMARY_KEY_END_BLOCK, ) from ..publish import publish_json from ..settings import MOONSTREAM_IPC_PATH @@ -62,8 +65,32 @@ def get_latest_block_from_db(db_session: Session): ) -def get_latest_summary_block() -> Optional[int]: - return None +# TODO move to sync handler +def get_latest_summary_block( + bugout_access_token: str, bugout_journal_id: str +) -> Optional[int]: + try: + bugout_client = Bugout() + query = "#crawl_type:nft_ethereum" + + events = bugout_client.search( + bugout_access_token, + bugout_journal_id, + query, + limit=1, + timeout=30.0, + order=SearchOrder.DESCENDING, + ) + if not events.results: + logger.warning("There is no summaries in Bugout") + return None + + last_event = events.results[0] + content = cast(str, last_event.content) + return json.loads(content)["end_block"] + except Exception as e: + logger.error(f"Failed to get summary from Bugout : {e}") + return None def get_latest_nft_labeled_block(db_session: Session) -> Optional[int]: @@ -93,7 +120,7 @@ def get_latest_nft_labeled_block(db_session: Session) -> Optional[int]: def sync_labels(db_session: Session, web3_client: Web3, start: Optional[int]) -> int: if start is None: logger.info( - "Syncing start block is not given, getting it from latest nft label in db" + "Syncing label start block is not given, getting it from latest nft label in db" ) start = get_latest_nft_labeled_block(db_session) if start is None: @@ -126,17 +153,19 @@ def sync_summaries( start: Optional[int], end: int, humbug_token: str, + bugout_access_token: str, + bugout_journal_id: str, ) -> int: if start is None: logger.info( - "Syncing start time is not given, getting it from latest nft label in db" + "Syncing summary start block is not given, getting it from latest nft summary from Bugout" ) - start = get_latest_summary_block() + start = get_latest_summary_block(bugout_access_token, bugout_journal_id) if start is not None: start += 1 else: logger.info( - "There is no entry in humbug, starting to create summaries from 3 month ago" + "There is no entry in Bugout, starting to create summaries from 3 month ago" ) time_now = datetime.now(timezone.utc) start_date = time_now - relativedelta(months=3) @@ -171,21 +200,37 @@ def ethereum_sync_handler(args: argparse.Namespace) -> None: humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") if humbug_token is None: raise ValueError("MOONSTREAM_HUMBUG_TOKEN env variable is not set") + bugout_access_token = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN") + if bugout_access_token is None: + raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN env variable is not set") + bugout_journal_id = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID") + if bugout_journal_id is None: + raise ValueError("MOONSTREAM_DATA_JOURNAL_ID env variable is not set") with yield_db_session_ctx() as db_session: logger.info("Initial labeling:") last_labeled = sync_labels(db_session, web3_client, args.start) logger.info("Initial summary creation:") last_summary_created = sync_summaries( - db_session, args.start, last_labeled, humbug_token + db_session, + args.start, + last_labeled, + humbug_token, + bugout_access_token, + bugout_journal_id, ) while True: logger.info("Syncing") last_labeled = sync_labels(db_session, web3_client, last_labeled + 1) last_summary_created = sync_summaries( - db_session, last_summary_created + 1, last_labeled, humbug_token + db_session, + last_summary_created + 1, + last_labeled, + humbug_token, + bugout_access_token, + bugout_journal_id, ) - sleep_time = 20 * 60 + sleep_time = 6 * 60 logger.info(f"Going to sleep for {sleep_time}s") time.sleep(sleep_time) @@ -198,12 +243,14 @@ def ethereum_label_handler(args: argparse.Namespace) -> None: def push_summary(result: Dict[str, Any], humbug_token: str): title = ( - f"NFT activity on the Ethereum blockchain: end time: {result['crawled_at'] })" + f"NFT activity on the Ethereum blockchain: crawled at: {result['crawled_at'] })" ) tags = [ f"crawler_version:{MOONCRAWL_VERSION}", f"summary_id:{result.get(SUMMARY_KEY_ID, '')}", + f"start_block:{result.get(SUMMARY_KEY_START_BLOCK)}", + f"end_block:{result.get(SUMMARY_KEY_END_BLOCK)}", ] # Add an "error:missing_blocks" tag for all summaries in which the number of blocks processed From 8d11e1c8f0051fc6c08db7b693b062ab9a2a0269 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 21 Sep 2021 18:32:32 +0300 Subject: [PATCH 13/24] removed comment and extended sleep time to 10 minutes --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index cf50407e..4afc2fcb 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -65,7 +65,6 @@ def get_latest_block_from_db(db_session: Session): ) -# TODO move to sync handler def get_latest_summary_block( bugout_access_token: str, bugout_journal_id: str ) -> Optional[int]: @@ -230,7 +229,7 @@ def ethereum_sync_handler(args: argparse.Namespace) -> None: bugout_access_token, bugout_journal_id, ) - sleep_time = 6 * 60 + sleep_time = 10 * 60 logger.info(f"Going to sleep for {sleep_time}s") time.sleep(sleep_time) From 94e7069c8af8c63f503d6909b30fb73ca39d5499 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Tue, 21 Sep 2021 18:47:13 +0300 Subject: [PATCH 14/24] changed default time for sync to 1st jan of 20201 --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index 4afc2fcb..393429fc 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -126,8 +126,7 @@ def sync_labels(db_session: Session, web3_client: Web3, start: Optional[int]) -> logger.warning( "Didn't find any nft labels in db, starting sync from 3 month before now" ) - time_now = datetime.now(timezone.utc) - start_date = time_now - relativedelta(months=3) + start_date = datetime(2021, 1, 1, tzinfo=timezone.utc) start = ( db_session.query(EthereumBlock) .filter(EthereumBlock.timestamp >= start_date.timestamp()) @@ -166,8 +165,7 @@ def sync_summaries( logger.info( "There is no entry in Bugout, starting to create summaries from 3 month ago" ) - time_now = datetime.now(timezone.utc) - start_date = time_now - relativedelta(months=3) + start_date = datetime(2021, 1, 1, tzinfo=timezone.utc) start = ( db_session.query(EthereumBlock) .filter(EthereumBlock.timestamp >= start_date.timestamp()) From d8cfa5a9967d6ad794bfd5e83b195fcbb6c35f14 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Wed, 22 Sep 2021 11:37:07 +0300 Subject: [PATCH 15/24] moved env variables to settings.py --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 56 +++++++----------------- crawlers/mooncrawl/mooncrawl/settings.py | 14 ++++++ 2 files changed, 30 insertions(+), 40 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index 393429fc..ff58d03c 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -3,10 +3,8 @@ A command line tool to crawl information about NFTs from various sources. """ import argparse from datetime import datetime, timezone -from dateutil.relativedelta import relativedelta import json import logging -import os import sys import time from typing import Any, cast, Dict, Optional @@ -31,7 +29,12 @@ from .ethereum import ( SUMMARY_KEY_END_BLOCK, ) from ..publish import publish_json -from ..settings import MOONSTREAM_IPC_PATH +from ..settings import ( + MOONSTREAM_IPC_PATH, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_HUMBUG_TOKEN, + MOONSTREAM_DATA_JOURNAL_ID, +) from ..version import MOONCRAWL_VERSION logging.basicConfig(level=logging.INFO) @@ -65,11 +68,12 @@ def get_latest_block_from_db(db_session: Session): ) -def get_latest_summary_block( - bugout_access_token: str, bugout_journal_id: str -) -> Optional[int]: +def get_latest_summary_block() -> Optional[int]: try: + bugout_client = Bugout() + bugout_access_token = cast(str, MOONSTREAM_ADMIN_ACCESS_TOKEN) + bugout_journal_id = cast(str, MOONSTREAM_DATA_JOURNAL_ID) query = "#crawl_type:nft_ethereum" events = bugout_client.search( @@ -150,15 +154,12 @@ def sync_summaries( db_session: Session, start: Optional[int], end: int, - humbug_token: str, - bugout_access_token: str, - bugout_journal_id: str, ) -> int: if start is None: logger.info( "Syncing summary start block is not given, getting it from latest nft summary from Bugout" ) - start = get_latest_summary_block(bugout_access_token, bugout_journal_id) + start = get_latest_summary_block() if start is not None: start += 1 else: @@ -180,7 +181,7 @@ def sync_summaries( logger.warn("Syncing summaries is not required") while batch_end <= end: summary_result = ethereum_summary(db_session, start, batch_end) - push_summary(summary_result, humbug_token) + push_summary(summary_result) logger.info(f"Pushed summary of blocks : {start}-{batch_end}") start = batch_end + 1 batch_end += BLOCKS_PER_SUMMARY @@ -194,16 +195,6 @@ def sync_summaries( def ethereum_sync_handler(args: argparse.Namespace) -> None: web3_client = web3_client_from_cli_or_env(args) - humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") - if humbug_token is None: - raise ValueError("MOONSTREAM_HUMBUG_TOKEN env variable is not set") - bugout_access_token = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN") - if bugout_access_token is None: - raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN env variable is not set") - bugout_journal_id = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID") - if bugout_journal_id is None: - raise ValueError("MOONSTREAM_DATA_JOURNAL_ID env variable is not set") - with yield_db_session_ctx() as db_session: logger.info("Initial labeling:") last_labeled = sync_labels(db_session, web3_client, args.start) @@ -212,9 +203,6 @@ def ethereum_sync_handler(args: argparse.Namespace) -> None: db_session, args.start, last_labeled, - humbug_token, - bugout_access_token, - bugout_journal_id, ) while True: logger.info("Syncing") @@ -223,9 +211,6 @@ def ethereum_sync_handler(args: argparse.Namespace) -> None: db_session, last_summary_created + 1, last_labeled, - humbug_token, - bugout_access_token, - bugout_journal_id, ) sleep_time = 10 * 60 logger.info(f"Going to sleep for {sleep_time}s") @@ -238,7 +223,9 @@ def ethereum_label_handler(args: argparse.Namespace) -> None: add_labels(web3_client, db_session, args.start, args.end, args.address) -def push_summary(result: Dict[str, Any], humbug_token: str): +def push_summary(result: Dict[str, Any], humbug_token: Optional[str] = None): + if humbug_token is None: + humbug_token = cast(str, MOONSTREAM_HUMBUG_TOKEN) title = ( f"NFT activity on the Ethereum blockchain: crawled at: {result['crawled_at'] })" ) @@ -287,18 +274,12 @@ def ethereum_summary_handler(args: argparse.Namespace) -> None: with yield_db_session_ctx() as db_session: result = ethereum_summary(db_session, args.start, args.end) - humbug_token = args.humbug - if humbug_token is None: - humbug_token = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") - if humbug_token: - push_summary(result, humbug_token) + push_summary(result, args.humbug) with args.outfile as ofp: json.dump(result, ofp) def main() -> None: - time_now = datetime.now(timezone.utc) - parser = argparse.ArgumentParser(description="Moonstream NFT crawlers") parser.set_defaults(func=lambda _: parser.print_help()) subcommands = parser.add_subparsers(description="Subcommands") @@ -390,11 +371,6 @@ def main() -> None: required=False, help="Starting block number (inclusive if block available)", ) - parser_ethereum_sync.add_argument( - "--humbug", - default=None, - help=("Humbug token to publish summary reports"), - ) parser_ethereum_sync.set_defaults(func=ethereum_sync_handler) args = parser.parse_args() diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 307f2e8a..e007f127 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -1,4 +1,5 @@ import os +from typing import cast # Bugout HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN") @@ -18,3 +19,16 @@ except: # Etherscan MOONSTREAM_ETHERSCAN_TOKEN = os.environ.get("MOONSTREAM_ETHERSCAN_TOKEN") + +# NFT crawler +MOONSTREAM_HUMBUG_TOKEN = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") +if MOONSTREAM_HUMBUG_TOKEN is None: + raise ValueError("MOONSTREAM_HUMBUG_TOKEN env variable is not set") + +MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN") +if MOONSTREAM_ADMIN_ACCESS_TOKEN is None: + raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN env variable is not set") + +MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID") +if MOONSTREAM_DATA_JOURNAL_ID is None: + raise ValueError("MOONSTREAM_DATA_JOURNAL_ID env variable is not set") From db9e35e9548174348db4aa9b3af63018578bc49b Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Wed, 22 Sep 2021 11:43:41 +0300 Subject: [PATCH 16/24] added new env variables to samle.env --- crawlers/mooncrawl/sample.env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 5d4ac548..6930735b 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -7,3 +7,5 @@ export AWS_S3_SMARTCONTRACT_BUCKET="" export MOONSTREAM_HUMBUG_TOKEN="" export COINMARKETCAP_API_KEY="" export HUMBUG_REPORTER_CRAWLERS_TOKEN="" +export MOONSTREAM_DATA_JOURNAL_ID="" +export MOONSTREAM_ADMIN_ACCESS_TOKEN="" \ No newline at end of file From 70a75b8d698c68cec0ac8b9adb176618e0b5fc7f Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Wed, 22 Sep 2021 13:02:47 +0300 Subject: [PATCH 17/24] fixed mypy issue --- backend/moonstream/providers/bugout.py | 30 +++++++++++++++++++++++++- backend/moonstream/routes/nft.py | 6 +++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/backend/moonstream/providers/bugout.py b/backend/moonstream/providers/bugout.py index 4a3d280b..ee96698d 100644 --- a/backend/moonstream/providers/bugout.py +++ b/backend/moonstream/providers/bugout.py @@ -321,6 +321,34 @@ class EthereumTXPoolProvider(BugoutEventProvider): return subscriptions_filters +class NftProvider(BugoutEventProvider): + def __init__( + self, + event_type: str, + description: str, + default_time_interval_seconds: int, + estimated_events_per_time_interval: float, + tags: Optional[List[str]] = None, + batch_size: int = 100, + timeout: float = 30.0, + ): + + super().__init__( + event_type=event_type, + description=description, + default_time_interval_seconds=default_time_interval_seconds, + estimated_events_per_time_interval=estimated_events_per_time_interval, + tags=tags, + batch_size=batch_size, + timeout=timeout, + ) + + def parse_filters( + self, query: StreamQuery, user_subscriptions: Dict[str, List[BugoutResource]] + ) -> Optional[List[str]]: + return [] + + whalewatch_description = """Event provider for Ethereum whale watch. Shows the top 10 addresses active on the Ethereum blockchain over the last hour in the following categories: @@ -360,7 +388,7 @@ Currently, it summarizes the activities on the following NFT markets: This provider is currently not accessible for subscription. The data from this provider is publicly available at the /nft endpoint.""" -nft_summary_provider = BugoutEventProvider( +nft_summary_provider = NftProvider( event_type="nft_summary", description=nft_summary_description, # 40 blocks per summary, 15 seconds per block + 2 seconds wiggle room. diff --git a/backend/moonstream/routes/nft.py b/backend/moonstream/routes/nft.py index 5bc849ed..c5eda166 100644 --- a/backend/moonstream/routes/nft.py +++ b/backend/moonstream/routes/nft.py @@ -7,6 +7,8 @@ from datetime import datetime import logging from typing import Optional +from bugout.data import BugoutResource + from fastapi import Depends, FastAPI, Query from moonstreamdb import db from fastapi.middleware.cors import CORSMiddleware @@ -70,9 +72,7 @@ async def stream_handler( data_journal_id=MOONSTREAM_DATA_JOURNAL_ID, data_access_token=MOONSTREAM_ADMIN_ACCESS_TOKEN, stream_boundary=stream_boundary, - user_subscriptions={ - nft_summary_provider.event_type: [nft_summary_provider.event_type] - }, + user_subscriptions={nft_summary_provider.event_type: []}, query=StreamQuery(subscription_types=[nft_summary_provider.event_type]), ) From 310b23501eaa4ddc618e7e4c1ec837c5c20b23ae Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Wed, 22 Sep 2021 13:08:29 +0300 Subject: [PATCH 18/24] added bugout to setup.py --- crawlers/mooncrawl/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 8e7eda49..6d00992b 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -32,13 +32,14 @@ setup( package_data={"mooncrawl": ["py.typed"]}, zip_safe=False, install_requires=[ + "boto3", + "bugout >= 0.1.17", "moonstreamdb @ git+https://git@github.com/bugout-dev/moonstream.git@a4fff6498f66789934d4af26fd42a8cfb6e5eed5#egg=moonstreamdb&subdirectory=db", "humbug", "python-dateutil", "requests", "tqdm", "web3", - "boto3", ], extras_require={ "dev": ["black", "mypy", "types-requests", "types-python-dateutil"] From f27d828d5a93e23eb7103143ca6e3c16b91631a1 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Wed, 22 Sep 2021 16:37:27 +0300 Subject: [PATCH 19/24] small fixes --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index ff58d03c..f667a11a 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -117,7 +117,11 @@ def get_latest_nft_labeled_block(db_session: Session) -> Optional[int]: .limit(1) ) - return query.one_or_none().block_number + start_block = query.one_or_none() + if start_block is not None: + return start_block.block_number + else: + return None def sync_labels(db_session: Session, web3_client: Web3, start: Optional[int]) -> int: @@ -128,7 +132,7 @@ def sync_labels(db_session: Session, web3_client: Web3, start: Optional[int]) -> start = get_latest_nft_labeled_block(db_session) if start is None: logger.warning( - "Didn't find any nft labels in db, starting sync from 3 month before now" + "Didn't find any nft labels in db, starting sync from 1st Jan 2021 before now" ) start_date = datetime(2021, 1, 1, tzinfo=timezone.utc) start = ( @@ -164,7 +168,7 @@ def sync_summaries( start += 1 else: logger.info( - "There is no entry in Bugout, starting to create summaries from 3 month ago" + "There is no entry in Bugout, starting to create summaries from 1st Jan 2021" ) start_date = datetime(2021, 1, 1, tzinfo=timezone.utc) start = ( From e75fcb7c220c40220c5e8d342c9c56df45746ac4 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Wed, 22 Sep 2021 16:53:11 +0300 Subject: [PATCH 20/24] removed casting for env variables. --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 8 +++----- crawlers/mooncrawl/mooncrawl/settings.py | 12 ++++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index f667a11a..a99a0c32 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -72,13 +72,11 @@ def get_latest_summary_block() -> Optional[int]: try: bugout_client = Bugout() - bugout_access_token = cast(str, MOONSTREAM_ADMIN_ACCESS_TOKEN) - bugout_journal_id = cast(str, MOONSTREAM_DATA_JOURNAL_ID) query = "#crawl_type:nft_ethereum" events = bugout_client.search( - bugout_access_token, - bugout_journal_id, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_DATA_JOURNAL_ID, query, limit=1, timeout=30.0, @@ -229,7 +227,7 @@ def ethereum_label_handler(args: argparse.Namespace) -> None: def push_summary(result: Dict[str, Any], humbug_token: Optional[str] = None): if humbug_token is None: - humbug_token = cast(str, MOONSTREAM_HUMBUG_TOKEN) + humbug_token = MOONSTREAM_HUMBUG_TOKEN title = ( f"NFT activity on the Ethereum blockchain: crawled at: {result['crawled_at'] })" ) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index e007f127..634397c4 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -21,14 +21,14 @@ except: MOONSTREAM_ETHERSCAN_TOKEN = os.environ.get("MOONSTREAM_ETHERSCAN_TOKEN") # NFT crawler -MOONSTREAM_HUMBUG_TOKEN = os.environ.get("MOONSTREAM_HUMBUG_TOKEN") -if MOONSTREAM_HUMBUG_TOKEN is None: +MOONSTREAM_HUMBUG_TOKEN = os.environ.get("MOONSTREAM_HUMBUG_TOKEN", "") +if MOONSTREAM_HUMBUG_TOKEN == "": raise ValueError("MOONSTREAM_HUMBUG_TOKEN env variable is not set") -MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN") -if MOONSTREAM_ADMIN_ACCESS_TOKEN is None: +MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN", "") +if MOONSTREAM_ADMIN_ACCESS_TOKEN == "": raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN env variable is not set") -MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID") -if MOONSTREAM_DATA_JOURNAL_ID is None: +MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID", "") +if MOONSTREAM_DATA_JOURNAL_ID == "": raise ValueError("MOONSTREAM_DATA_JOURNAL_ID env variable is not set") From 711319fbadf138bbd293dc48d22ab2aba796d6b4 Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Thu, 23 Sep 2021 12:02:14 +0300 Subject: [PATCH 21/24] added humbug reporter --- crawlers/mooncrawl/mooncrawl/nft/ethereum.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py index c42d0ea4..0b0e3fd5 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/ethereum.py +++ b/crawlers/mooncrawl/mooncrawl/nft/ethereum.py @@ -20,6 +20,8 @@ from web3 import Web3 from web3.types import FilterParams, LogReceipt from web3._utils.events import get_event_data +from ..reporter import reporter + # Default length (in blocks) of an Ethereum NFT crawl. DEFAULT_CRAWL_LENGTH = 100 @@ -317,6 +319,7 @@ def label_erc721_addresses( db_session.bulk_save_objects(labels) db_session.commit() except Exception as e: + reporter.error_report(e, ["nft-crawler"], True) db_session.rollback() logger.error(f"Failed to save labels to db:\n{e}") @@ -372,6 +375,7 @@ def label_transfers( db_session.bulk_save_objects(new_labels) db_session.commit() except Exception as e: + reporter.error_report(e, ["nft-crawler"], True) db_session.rollback() logger.error("Could not write transfer/mint labels to database") logger.error(e) @@ -383,7 +387,7 @@ def add_labels( from_block: Optional[int] = None, to_block: Optional[int] = None, contract_address: Optional[str] = None, - batch_size: int = 50, + batch_size: int = 100, ) -> None: """ Crawls blocks between from_block and to_block checking for NFT mints and transfers. From 54ebdd922b066c375330122ce41d4f1d8bc1f2ad Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Thu, 23 Sep 2021 13:08:06 +0300 Subject: [PATCH 22/24] MOONSTREAM_HUMBUG_TOKEN -> NFT_HUMBUG_TOKEN --- crawlers/mooncrawl/mooncrawl/nft/cli.py | 4 ++-- crawlers/mooncrawl/sample.env | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/nft/cli.py b/crawlers/mooncrawl/mooncrawl/nft/cli.py index a99a0c32..a2934b37 100644 --- a/crawlers/mooncrawl/mooncrawl/nft/cli.py +++ b/crawlers/mooncrawl/mooncrawl/nft/cli.py @@ -32,7 +32,7 @@ from ..publish import publish_json from ..settings import ( MOONSTREAM_IPC_PATH, MOONSTREAM_ADMIN_ACCESS_TOKEN, - MOONSTREAM_HUMBUG_TOKEN, + NFT_HUMBUG_TOKEN, MOONSTREAM_DATA_JOURNAL_ID, ) from ..version import MOONCRAWL_VERSION @@ -227,7 +227,7 @@ def ethereum_label_handler(args: argparse.Namespace) -> None: def push_summary(result: Dict[str, Any], humbug_token: Optional[str] = None): if humbug_token is None: - humbug_token = MOONSTREAM_HUMBUG_TOKEN + humbug_token = NFT_HUMBUG_TOKEN title = ( f"NFT activity on the Ethereum blockchain: crawled at: {result['crawled_at'] })" ) diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 6930735b..a49b2bfb 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -8,4 +8,5 @@ export MOONSTREAM_HUMBUG_TOKEN="" export COINMARKETCAP_API_KEY="" export HUMBUG_REPORTER_CRAWLERS_TOKEN="" export MOONSTREAM_DATA_JOURNAL_ID="" -export MOONSTREAM_ADMIN_ACCESS_TOKEN="" \ No newline at end of file +export MOONSTREAM_ADMIN_ACCESS_TOKEN="" +export NFT_HUMBUG_TOKEN="" \ No newline at end of file From 1e7a070335b62cf78b4f8b88751695049da0f34c Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Thu, 23 Sep 2021 13:13:06 +0300 Subject: [PATCH 23/24] NFT_HUMBUG_TOKEN --- crawlers/mooncrawl/mooncrawl/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 634397c4..00a8fbd9 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -21,9 +21,9 @@ except: MOONSTREAM_ETHERSCAN_TOKEN = os.environ.get("MOONSTREAM_ETHERSCAN_TOKEN") # NFT crawler -MOONSTREAM_HUMBUG_TOKEN = os.environ.get("MOONSTREAM_HUMBUG_TOKEN", "") -if MOONSTREAM_HUMBUG_TOKEN == "": - raise ValueError("MOONSTREAM_HUMBUG_TOKEN env variable is not set") +NFT_HUMBUG_TOKEN = os.environ.get("NFT_HUMBUG_TOKEN", "") +if NFT_HUMBUG_TOKEN == "": + raise ValueError("NFT_HUMBUG_TOKEN env variable is not set") MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN", "") if MOONSTREAM_ADMIN_ACCESS_TOKEN == "": From 20a71897061a8a1a7741cff7692a5281e2d4354f Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Thu, 23 Sep 2021 13:14:01 +0300 Subject: [PATCH 24/24] removed nft service --- crawlers/deploy/ethereum-nft.service | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 crawlers/deploy/ethereum-nft.service diff --git a/crawlers/deploy/ethereum-nft.service b/crawlers/deploy/ethereum-nft.service deleted file mode 100644 index abd0b1bc..00000000 --- a/crawlers/deploy/ethereum-nft.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Labels nft transactions and sends summary to humbug -After=network.target - -[Service] -Type=oneshot -User=ubuntu -Group=www-data -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/mooncrawl-secrets/app.env -ExecStart=/usr/bin/bash -c '/home/ubuntu/mooncrawl-env/bin/python -m mooncrawl.nft.cli ethereum synchronize'