Merge pull request #616 from bugout-dev/xdai-subscription

XDai subscriptions
pull/621/head
Sergei Sumarokov 2022-05-26 17:27:05 +03:00 zatwierdzone przez GitHub
commit c350539aad
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
19 zmienionych plików z 116 dodań i 41 usunięć

Wyświetl plik

@ -48,6 +48,8 @@ blockchain_by_subscription_id = {
"polygon_blockchain": "polygon",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"xdai_blockchain": "xdai",
"xdai_smartcontract": "xdai",
}

Wyświetl plik

@ -238,7 +238,7 @@ This CLI is configured to work with the following API URLs:
)
parser_subscription_types_update = subcommands_subscription_types.add_parser(
"update", description="Create subscription type"
"update", description="Update subscription type"
)
parser_subscription_types_update.add_argument(
"-i", "--id", required=True, type=str, help="ID for the subscription type"

Wyświetl plik

@ -37,6 +37,16 @@ CANONICAL_SUBSCRIPTION_TYPES = {
stripe_price_id=None,
active=True,
),
"xdai_smartcontract": SubscriptionTypeResourceData(
id="xdai_smartcontract",
name="XDai smartcontract",
choices=["input:address", "tag:erc721"],
description="Contracts events and tx_calls of contract of XDai blockchain.",
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/xdai-token-logo.png",
stripe_product_id=None,
stripe_price_id=None,
active=True,
),
"ethereum_blockchain": SubscriptionTypeResourceData(
id="ethereum_blockchain",
name="Ethereum transactions",
@ -57,6 +67,16 @@ CANONICAL_SUBSCRIPTION_TYPES = {
stripe_price_id=None,
active=True,
),
"xdai_blockchain": SubscriptionTypeResourceData(
id="xdai_blockchain",
name="XDai transactions",
choices=["input:address", "tag:erc721"],
description="Gnosis chain transactions subscription.",
icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/xdai-token-logo.png",
stripe_product_id=None,
stripe_price_id=None,
active=True,
),
"ethereum_whalewatch": SubscriptionTypeResourceData(
id="ethereum_whalewatch",
name="Ethereum whale watch",

Wyświetl plik

@ -49,8 +49,10 @@ class ReceivingEventsException(Exception):
event_providers: Dict[str, Any] = {
moonworm_provider.EthereumMoonwormProvider.event_type: moonworm_provider.EthereumMoonwormProvider,
moonworm_provider.PolygonMoonwormProvider.event_type: moonworm_provider.PolygonMoonwormProvider,
moonworm_provider.XDaiMoonwormProvider.event_type: moonworm_provider.XDaiMoonwormProvider,
transactions.EthereumTransactions.event_type: transactions.EthereumTransactions,
transactions.PolygonTransactions.event_type: transactions.PolygonTransactions,
transactions.XDaiTransactions.event_type: transactions.XDaiTransactions,
bugout.polygon_whalewatch_provider.event_type: bugout.polygon_whalewatch_provider,
bugout.ethereum_txpool_provider.event_type: bugout.ethereum_txpool_provider,
bugout.ethereum_whalewatch_provider.event_type: bugout.ethereum_whalewatch_provider,

Wyświetl plik

@ -19,6 +19,7 @@ logger.setLevel(logging.WARN)
ethereum_event_type = "ethereum_blockchain"
polygon_event_type = "polygon_blockchain"
xdai_event_type = "xdai_blockchain"
allowed_tags = ["tag:erc721"]
description = f"""Event provider for transactions from the Ethereum blockchain.
@ -402,3 +403,10 @@ PolygonMoonwormProvider = MoonwormProvider(
description="Provider for resiving transactions from Polygon tables.",
streamboaundary_range_limit=2 * 60 * 60,
)
XDaiMoonwormProvider = MoonwormProvider(
event_type="xdai_smartcontract",
blockchain=AvailableBlockchainType("xdai"),
description="Provider for resiving transactions from XDai tables.",
streamboaundary_range_limit=2 * 60 * 60,
)

Wyświetl plik

@ -461,3 +461,10 @@ PolygonTransactions = TransactionsProvider(
description="Provider for resiving transactions from Polygon tables.",
streamboaundary_range_limit=2 * 60 * 60,
)
XDaiTransactions = TransactionsProvider(
event_type="xdai_blockchain",
blockchain=AvailableBlockchainType("xdai"),
description="Provider for resiving transactions from XDai tables.",
streamboaundary_range_limit=2 * 60 * 60,
)

Wyświetl plik

@ -34,14 +34,6 @@ BUGOUT_RESOURCE_TYPE_DASHBOARD = "dashboards"
BUGOUT_RESOURCE_TYPE_SUBSCRIPTION = "subscription"
blockchain_by_subscription_id = {
"ethereum_blockchain": "ethereum",
"polygon_blockchain": "polygon",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
}
@router.post("/", tags=["dashboards"], response_model=BugoutResource)
async def add_dashboard_handler(
request: Request,
@ -419,7 +411,7 @@ async def get_dashboard_data_links_handler(
stats[subscription.id] = {}
for timescale in available_timescales:
try:
result_key = f'{MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX}/{blockchain_by_subscription_id[subscription.resource_data["subscription_type_id"]]}/contracts_data/{subscription.resource_data["address"]}/{dashboard_id}/v1/{timescale}.json'
result_key = f'{MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX}/{actions.blockchain_by_subscription_id[subscription.resource_data["subscription_type_id"]]}/contracts_data/{subscription.resource_data["address"]}/{dashboard_id}/v1/{timescale}.json'
stats_presigned_url = s3_client.generate_presigned_url(
"get_object",
Params={

Wyświetl plik

@ -2,4 +2,4 @@
Moonstream library and API version.
"""
MOONSTREAMAPI_VERSION = "0.1.3"
MOONSTREAMAPI_VERSION = "0.1.4"

Wyświetl plik

@ -15,7 +15,7 @@ setup(
"boto3",
"bugout>=0.1.19",
"fastapi",
"moonstreamdb>=0.2.3",
"moonstreamdb>=0.2.5",
"humbug",
"pydantic",
"pyevmasm",

Wyświetl plik

@ -50,6 +50,8 @@ POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service"
XDAI_SYNCHRONIZE_SERVICE="xdai-synchronize.service"
XDAI_MISSING_SERVICE_FILE="xdai-missing.service"
XDAI_MISSING_TIMER_FILE="xdai-missing.timer"
XDAI_STATISTICS_SERVICE_FILE="xdai-statistics.service"
XDAI_STATISTICS_TIMER_FILE="xdai-statistics.timer"
set -eu
@ -98,7 +100,7 @@ echo -e "${PREFIX_INFO} Replacing existing Moonstream crawlers HTTP API server s
chmod 644 "${SCRIPT_DIR}/${MOONCRAWL_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${MOONCRAWL_SERVICE_FILE}" "/etc/systemd/system/${MOONCRAWL_SERVICE_FILE}"
systemctl daemon-reload
systemctl restart "${MOONCRAWL_SERVICE_FILE}"
systemctl restart --no-block "${MOONCRAWL_SERVICE_FILE}"
echo
echo
@ -106,7 +108,7 @@ echo -e "${PREFIX_INFO} Replacing existing Ethereum block with transactions sync
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}"
systemctl daemon-reload
systemctl restart "${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}"
systemctl restart --no-block "${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}"
echo
echo
@ -115,7 +117,7 @@ chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETH
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_TRENDING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/etc/systemd/system/${ETHEREUM_TRENDING_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${ETHEREUM_TRENDING_TIMER_FILE}"
systemctl restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}"
# echo
# echo
@ -132,7 +134,7 @@ chmod 644 "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHE
cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_SERVICE_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_MISSING_TIMER_FILE}" "/etc/systemd/system/${ETHEREUM_MISSING_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${ETHEREUM_MISSING_TIMER_FILE}"
systemctl restart --no-block "${ETHEREUM_MISSING_TIMER_FILE}"
echo
echo
@ -140,7 +142,7 @@ echo -e "${PREFIX_INFO} Replacing existing Polygon block with transactions syncr
chmod 644 "${SCRIPT_DIR}/${POLYGON_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${POLYGON_SYNCHRONIZE_SERVICE}" "/etc/systemd/system/${POLYGON_SYNCHRONIZE_SERVICE}"
systemctl daemon-reload
systemctl restart "${POLYGON_SYNCHRONIZE_SERVICE}"
systemctl restart --no-block "${POLYGON_SYNCHRONIZE_SERVICE}"
echo
echo
@ -149,7 +151,7 @@ chmod 644 "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYG
cp "${SCRIPT_DIR}/${POLYGON_MISSING_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}" "/etc/systemd/system/${POLYGON_MISSING_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${POLYGON_MISSING_TIMER_FILE}"
systemctl restart --no-block "${POLYGON_MISSING_TIMER_FILE}"
echo
echo
@ -166,7 +168,7 @@ systemctl restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}"
# chmod 644 "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}"
# cp "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_TXPOOL_SERVICE_FILE}"
# systemctl daemon-reload
# systemctl restart "${POLYGON_TXPOOL_SERVICE_FILE}"
# systemctl restart --no-block "${POLYGON_TXPOOL_SERVICE_FILE}"
echo
echo
@ -174,7 +176,7 @@ echo -e "${PREFIX_INFO} Replacing existing Polygon moonworm crawler service defi
chmod 644 "${SCRIPT_DIR}/${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}"
systemctl daemon-reload
systemctl restart "${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}"
systemctl restart --no-block "${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}"
echo
echo
@ -182,7 +184,7 @@ echo -e "${PREFIX_INFO} Replacing existing XDai block with transactions syncroni
chmod 644 "${SCRIPT_DIR}/${XDAI_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${XDAI_SYNCHRONIZE_SERVICE}" "/etc/systemd/system/${XDAI_SYNCHRONIZE_SERVICE}"
systemctl daemon-reload
systemctl restart "${XDAI_SYNCHRONIZE_SERVICE}"
systemctl restart --no-block "${XDAI_SYNCHRONIZE_SERVICE}"
echo
echo
@ -191,4 +193,13 @@ chmod 644 "${SCRIPT_DIR}/${XDAI_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${XDAI_MIS
cp "${SCRIPT_DIR}/${XDAI_MISSING_SERVICE_FILE}" "/etc/systemd/system/${XDAI_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${XDAI_MISSING_TIMER_FILE}" "/etc/systemd/system/${XDAI_MISSING_TIMER_FILE}"
systemctl daemon-reload
systemctl restart "${XDAI_MISSING_TIMER_FILE}"
systemctl restart --no-block "${XDAI_MISSING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing XDai statistics dashbord service and timer with: ${XDAI_STATISTICS_SERVICE_FILE}, ${XDAI_STATISTICS_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${XDAI_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${XDAI_STATISTICS_TIMER_FILE}"
cp "${SCRIPT_DIR}/${XDAI_STATISTICS_SERVICE_FILE}" "/etc/systemd/system/${XDAI_STATISTICS_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${XDAI_STATISTICS_TIMER_FILE}" "/etc/systemd/system/${XDAI_STATISTICS_TIMER_FILE}"
systemctl daemon-reload
systemctl restart --no-block "${XDAI_STATISTICS_TIMER_FILE}"

Wyświetl plik

@ -0,0 +1,14 @@
[Unit]
Description=Update XDai statistics dashboards
After=network.target
[Service]
Type=oneshot
User=ubuntu
Group=www-data
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard \
--access-id "${NB_CONTROLLER_ACCESS_ID}" \
generate --blockchain xdai
SyslogIdentifier=xdai-statistics

Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Update XDai statistics dashboards each 6 hours
[Timer]
OnBootSec=10s
OnUnitActiveSec=6h
[Install]
WantedBy=timers.target

Wyświetl plik

@ -2,6 +2,7 @@ import argparse
import json
import logging
from typing import Optional
from uuid import UUID
from moonstreamdb.db import yield_db_session_ctx
from web3 import Web3
@ -11,6 +12,7 @@ from mooncrawl.data import AvailableBlockchainType # type: ignore
from ..blockchain import connect
from .base import crawl, get_checkpoint, populate_with_events
from ..settings import NB_CONTROLLER_ACCESS_ID
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -38,7 +40,7 @@ def handle_nft_crawler(args: argparse.Namespace) -> None:
logger.info(
"No web3 provider URL provided, using default (blockchan.py: connect())"
)
web3 = connect(blockchain_type)
web3 = connect(blockchain_type, access_id=args.access_id)
else:
logger.info(f"Using web3 provider URL: {args.web3}")
web3 = Web3(
@ -90,7 +92,7 @@ def populate_with_erc20_transfers(args: argparse.Namespace) -> None:
logger.info(
"No web3 provider URL provided, using default (blockchan.py: connect())"
)
web3 = connect(blockchain_type)
web3 = connect(blockchain_type, access_id=args.access_id)
else:
logger.info(f"Using web3 provider URL: {args.web3}")
web3 = Web3(
@ -136,7 +138,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
logger.info(
"No web3 provider URL provided, using default (blockchan.py: connect())"
)
web3 = connect(blockchain_type)
web3 = connect(blockchain_type, access_id=args.access_id)
else:
logger.info(f"Using web3 provider URL: {args.web3}")
web3 = Web3(
@ -170,6 +172,13 @@ def main():
subparsers = parser.add_subparsers()
parser.add_argument(
"--access-id",
default=NB_CONTROLLER_ACCESS_ID,
type=UUID,
help="User access ID",
)
crawl_parser = subparsers.add_parser("crawl", help="Crawl with abi")
crawl_parser.add_argument(
"--blockchain_type",
@ -238,10 +247,7 @@ def main():
"--blockchain_type",
type=str,
required=True,
choices=[
"ethereum",
"polygon",
],
choices=[member.value for member in AvailableBlockchainType],
)
nft_crawler_parser.add_argument(
"--web3",
@ -293,10 +299,7 @@ def main():
"--blockchain_type",
type=str,
required=True,
choices=[
"ethereum",
"polygon",
],
choices=[member.value for member in AvailableBlockchainType],
)
erc20_populate_parser.add_argument(
"--web3",

Wyświetl plik

@ -12,6 +12,7 @@ from ..settings import MOONSTREAM_MOONWORM_TASKS_JOURNAL, NB_CONTROLLER_ACCESS_I
from .continuous_crawler import _retry_connect_web3, continuous_crawler
from .crawler import (
SubscriptionTypes,
blockchain_type_to_subscription_type,
get_crawl_job_entries,
make_event_crawl_jobs,
make_function_call_crawl_jobs,
@ -24,9 +25,12 @@ logger = logging.getLogger(__name__)
def handle_crawl(args: argparse.Namespace) -> None:
blockchain_type = AvailableBlockchainType(args.blockchain_type)
subscription_type = blockchain_type_to_subscription_type(blockchain_type)
initial_event_jobs = make_event_crawl_jobs(
get_crawl_job_entries(
SubscriptionTypes.POLYGON_BLOCKCHAIN,
subscription_type,
"event",
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
)
@ -35,7 +39,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
initial_function_call_jobs = make_function_call_crawl_jobs(
get_crawl_job_entries(
SubscriptionTypes.POLYGON_BLOCKCHAIN,
subscription_type,
"function",
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
)
@ -44,8 +48,6 @@ def handle_crawl(args: argparse.Namespace) -> None:
f"Initial function call crawl jobs count: {len(initial_function_call_jobs)}"
)
blockchain_type = AvailableBlockchainType(args.blockchain_type)
logger.info(f"Blockchain type: {blockchain_type.value}")
with yield_db_session_ctx() as db_session:
web3: Optional[Web3] = None

Wyświetl plik

@ -250,7 +250,6 @@ def continuous_crawler(
):
# Commiting to db
commit_session(db_session)
# Update heartbeat
heartbeat_template["last_block"] = end_block
heartbeat_template["current_time"] = _date_to_str(current_time)

Wyświetl plik

@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
class SubscriptionTypes(Enum):
POLYGON_BLOCKCHAIN = "polygon_smartcontract"
ETHEREUM_BLOCKCHAIN = "ethereum_smartcontract"
XDAI_BLOCKCHAIN = "xdai_smartcontract"
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
@ -99,6 +100,8 @@ def blockchain_type_to_subscription_type(
return SubscriptionTypes.ETHEREUM_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.POLYGON:
return SubscriptionTypes.POLYGON_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.XDAI:
return SubscriptionTypes.XDAI_BLOCKCHAIN
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

Wyświetl plik

@ -37,13 +37,16 @@ logger = logging.getLogger(__name__)
subscription_ids_by_blockchain = {
"ethereum": ["ethereum_blockchain", "ethereum_smartcontract"],
"polygon": ["polygon_blockchain", "polygon_smartcontract"],
"xdai": ["xdai_blockchain", "xdai_smartcontract"],
}
blockchain_by_subscription_id = {
"ethereum_blockchain": "ethereum",
"polygon_blockchain": "polygon",
"xdai_blockchain": "xdai",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"xdai_smartcontract": "xdai",
}

Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.1.5"
MOONCRAWL_VERSION = "0.1.6"

Wyświetl plik

@ -37,8 +37,8 @@ setup(
"bugout>=0.1.19",
"chardet",
"fastapi",
"moonstreamdb>=0.2.3",
"moonworm==0.1.11",
"moonstreamdb>=0.2.5",
"moonworm==0.2.4",
"humbug",
"pydantic",
"python-dateutil",