diff --git a/crawlers/deploy/arbitrum-nova-missing.service b/crawlers/deploy/arbitrum-nova-missing.service new file mode 100644 index 00000000..c2679270 --- /dev/null +++ b/crawlers/deploy/arbitrum-nova-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at Arbitrum Nova database +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_nova -n +CPUWeight=50 +SyslogIdentifier=arbitrum-nova-missing \ No newline at end of file diff --git a/crawlers/deploy/arbitrum-nova-missing.timer b/crawlers/deploy/arbitrum-nova-missing.timer new file mode 100644 index 00000000..e1ad1da8 --- /dev/null +++ b/crawlers/deploy/arbitrum-nova-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at Arbitrum Nova database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/arbitrum-nova-moonworm-crawler.service b/crawlers/deploy/arbitrum-nova-moonworm-crawler.service new file mode 100644 index 00000000..73accbd8 --- /dev/null +++ b/crawlers/deploy/arbitrum-nova-moonworm-crawler.service @@ -0,0 +1,17 @@ +[Unit] +Description=Arbitrum Nova moonworm crawler +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_nova --confirmations 10 --min-blocks-batch 20 +CPUWeight=70 +SyslogIdentifier=arbitrum-nova-moonworm-crawler + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/arbitrum-nova-synchronize.service b/crawlers/deploy/arbitrum-nova-synchronize.service new file mode 100644 index 00000000..fe569eb7 --- /dev/null +++ b/crawlers/deploy/arbitrum-nova-synchronize.service @@ -0,0 +1,17 @@ +[Unit] +Description=Arbitrum Nova block with transactions synchronizer +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_nova -c 10 -j 2 +CPUWeight=90 +SyslogIdentifier=arbitrum-nova-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 9f883381..28f323e4 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -122,6 +122,12 @@ ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="zksync-era-testnet- ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-testnet-historical-crawl-events.service" ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-testnet-historical-crawl-events.timer" +# Arbitrum Nova +ARBITRUM_NOVA_MISSING_SERVICE_FILE="arbitrum-nova-missing.service" +ARBITRUM_NOVA_MISSING_TIMER_FILE="arbitrum-nova-missing.timer" +ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-nova-moonworm-crawler.service" +ARBITRUM_NOVA_SYNCHRONIZE_SERVICE="arbitrum-nova-synchronize.service" + set -eu echo @@ -584,3 +590,29 @@ cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "/home/ubuntu/.config/sys cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}" + +# Arbitrum Nova +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova block with transactions syncronizer service definition with ${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova missing service and timer with: ${ARBITRUM_NOVA_MISSING_SERVICE_FILE}, ${ARBITRUM_NOVA_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_MISSING_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova moonworm crawler service definition with ${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" diff --git a/crawlers/deploy/monitoring-crawlers.service b/crawlers/deploy/monitoring-crawlers.service index 5a5aa09f..d1ce9dac 100644 --- a/crawlers/deploy/monitoring-crawlers.service +++ b/crawlers/deploy/monitoring-crawlers.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/ EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env -ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service +ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service CPUWeight=90 SyslogIdentifier=monitoring-crawlers diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index abb95ecc..ce045613 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -1,5 +1,5 @@ import logging -from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait +from concurrent.futures import Future, ThreadPoolExecutor, wait from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union from uuid import UUID @@ -21,6 +21,7 @@ from web3.types import BlockData from .data import DateRange from .db import yield_db_session, yield_db_session_ctx from .settings import ( + MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI, MOONSTREAM_CRAWL_WORKERS, MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI, MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI, @@ -76,6 +77,8 @@ def connect( web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI + elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: + web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI else: raise Exception("Wrong blockchain type provided for web3 URI") @@ -94,6 +97,15 @@ def connect( return web3_client +def hex_to_int(hex_str: Optional[str] = None) -> Optional[int]: + if hex_str is None: + return None + elif hex_str.startswith("0x"): + return int(hex_str, 16) + else: + return int(hex_str) + + def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) -> None: """ Add block if doesn't presented in database. @@ -145,6 +157,12 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) if block.get("l1BatchTimestamp") is not None else None ) + if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: + block_obj.sha3_uncles = block.get("sha3Uncles", "") + block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber")) + block_obj.send_count = hex_to_int(block.get("sendCount")) + block_obj.send_root = block.get("sendRoot", "") + block_obj.mix_hash = block.get("mixHash", "") db_session.add(block_obj) @@ -188,6 +206,8 @@ def add_block_transactions( if tx.get("l1BatchTxIndex") is not None else None ) + if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: + tx_obj.y_parity = hex_to_int(tx.get("yParity")) db_session.add(tx_obj) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index 0aac7927..9df1c5fc 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -6,13 +6,17 @@ from typing import Dict, List, Optional, Tuple from uuid import UUID from moonstreamdb.blockchain import AvailableBlockchainType +from moonstreamdb.networks import Network from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore MoonstreamEthereumStateProvider, ) -from moonstreamdb.networks import Network from sqlalchemy.orm.session import Session from web3 import Web3 +from ..settings import ( + HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES, + HISTORICAL_CRAWLER_STATUSES, +) from .crawler import ( EventCrawlJob, FunctionCallCrawlJob, @@ -29,10 +33,6 @@ from .crawler import ( from .db import add_events_to_session, add_function_calls_to_session, commit_session from .event_crawler import _crawl_events from .function_call_crawler import _crawl_functions -from ..settings import ( - HISTORICAL_CRAWLER_STATUSES, - HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES, -) logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -134,6 +134,8 @@ def continuous_crawler( network = Network.zksync_era_testnet elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: network = Network.zksync_era + elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: + network = Network.arbitrum_nova else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index fba881b5..a6473751 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -5,24 +5,24 @@ import time from dataclasses import dataclass from datetime import datetime from enum import Enum -from typing import Any, Callable, Dict, List, Optional, cast, Union, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast from uuid import UUID -from bugout.data import BugoutSearchResult, BugoutJournalEntries +from bugout.data import BugoutJournalEntries, BugoutSearchResult from eth_typing.evm import ChecksumAddress from moonstreamdb.blockchain import AvailableBlockchainType -from web3.main import Web3 from moonworm.deployment import find_deployment_block # type: ignore +from web3.main import Web3 from ..blockchain import connect from ..reporter import reporter from ..settings import ( BUGOUT_REQUEST_TIMEOUT_SECONDS, + HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES, + HISTORICAL_CRAWLER_STATUSES, MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_MOONWORM_TASKS_JOURNAL, bugout_client, - HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES, - HISTORICAL_CRAWLER_STATUSES, ) logging.basicConfig(level=logging.INFO) @@ -37,6 +37,7 @@ class SubscriptionTypes(Enum): WYRM_BLOCKCHAIN = "wyrm_smartcontract" ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract" ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract" + ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract" def abi_input_signature(input_abi: Dict[str, Any]) -> str: @@ -145,6 +146,8 @@ def blockchain_type_to_subscription_type( return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN + elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: + return SubscriptionTypes.ARBITRUM_NOVA_BLOCKCHAIN else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py index d5d650f8..a5c09f7f 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py @@ -2,6 +2,7 @@ import logging from typing import List from moonstreamdb.blockchain import AvailableBlockchainType +from moonstreamdb.networks import Network # type: ignore from moonworm.crawler.function_call_crawler import ( # type: ignore ContractFunctionCall, FunctionCallCrawler, @@ -9,7 +10,6 @@ from moonworm.crawler.function_call_crawler import ( # type: ignore from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore MoonstreamEthereumStateProvider, ) -from moonstreamdb.networks import Network # type: ignore from moonworm.watch import MockState # type: ignore from sqlalchemy.orm import Session from web3 import Web3 @@ -72,6 +72,8 @@ def function_call_crawler( network = Network.zksync_era_testnet elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: network = Network.zksync_era + elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: + network = Network.arbitrum_nova else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py index 24d7c33f..2db76fd0 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py @@ -5,10 +5,10 @@ from uuid import UUID from eth_typing.evm import ChecksumAddress from moonstreamdb.blockchain import AvailableBlockchainType +from moonstreamdb.networks import Network # type: ignore from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore MoonstreamEthereumStateProvider, ) -from moonstreamdb.networks import Network # type: ignore from sqlalchemy.orm.session import Session from web3 import Web3 @@ -19,7 +19,7 @@ from .crawler import ( update_entries_status_and_progress, ) from .db import add_events_to_session, add_function_calls_to_session, commit_session -from .event_crawler import _crawl_events, _autoscale_crawl_events +from .event_crawler import _autoscale_crawl_events, _crawl_events from .function_call_crawler import _crawl_functions logging.basicConfig(level=logging.INFO) @@ -65,6 +65,8 @@ def historical_crawler( network = Network.zksync_era_testnet elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: network = Network.zksync_era + elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: + network = Network.arbitrum_nova else: raise Exception("Unsupported blockchain type provided") diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 75079707..d9e6d74e 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -122,6 +122,14 @@ MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI = os.environ.get( if MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI == "": raise Exception("MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI env variable is not set") +MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI = os.environ.get( + "MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI", "" +) +if MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI == "": + raise Exception( + "MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI env variable is not set" + ) + MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") try: diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index c33dc632..2ecdac64 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -8,7 +8,7 @@ import logging import time from datetime import datetime, timedelta from enum import Enum -from typing import Any, Callable, cast, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Union, cast from uuid import UUID import boto3 # type: ignore @@ -23,8 +23,9 @@ from moonstreamdb.blockchain import ( get_label_model, get_transaction_model, ) -from sqlalchemy import and_, distinct, extract, func, text +from sqlalchemy import and_ from sqlalchemy import cast as sqlalchemy_cast +from sqlalchemy import distinct, extract, func, text from sqlalchemy.orm import Session from sqlalchemy.sql.operators import in_op from web3 import Web3 @@ -54,6 +55,7 @@ subscription_id_by_blockchain = { "wyrm": "wyrm_smartcontract", "zksync_era_testnet": "zksync_era_testnet_smartcontract", "zksync_era": "zksync_era_smartcontract", + "arbitrum_nova": "arbitrum_nova_smartcontract", } blockchain_by_subscription_id = { @@ -63,6 +65,7 @@ blockchain_by_subscription_id = { "xdai_blockchain": "xdai", "wyrm_blockchain": "wyrm", "zksync_era_testnet_blockchain": "zksync_era_testnet", + "arbitrum_nova_blockchain": "arbitrum_nova", "ethereum_smartcontract": "ethereum", "polygon_smartcontract": "polygon", "mumbai_smartcontract": "mumbai", @@ -70,6 +73,7 @@ blockchain_by_subscription_id = { "wyrm_smartcontract": "wyrm", "zksync_era_testnet_smartcontract": "zksync_era_testnet", "zksync_era_smartcontract": "zksync_era", + "arbitrum_nova_smartcontract": "arbitrum_nova", } diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index eea1a915..612a217e 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.3.5" +MOONCRAWL_VERSION = "0.3.6" diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index a02901a5..072fd0b2 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -27,6 +27,8 @@ export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://=0.2.13", "chardet", "fastapi", - "moonstreamdb>=0.3.5", + "moonstreamdb>=0.3.6", "moonstream>=0.1.1", "moonworm[moonstream]>=0.6.2", "humbug", diff --git a/moonstreamdb/alembic/env.py b/moonstreamdb/alembic/env.py index 6d9eb01f..708727af 100644 --- a/moonstreamdb/alembic/env.py +++ b/moonstreamdb/alembic/env.py @@ -46,6 +46,9 @@ from moonstreamdb.models import ( ZkSyncEraTestnetBlock, ZkSyncEraTestnetLabel, ZkSyncEraTestnetTransaction, + ArbitrumNovaBlock, + ArbitrumNovaTransaction, + ArbitrumNovaLabel, ) @@ -72,6 +75,9 @@ def include_symbol(tablename, schema): ZkSyncEraTestnetBlock.__tablename__, ZkSyncEraTestnetLabel.__tablename__, ZkSyncEraTestnetTransaction.__tablename__, + ArbitrumNovaBlock.__tablename__, + ArbitrumNovaTransaction.__tablename__, + ArbitrumNovaLabel.__tablename__, } diff --git a/moonstreamdb/alembic/versions/2391c3cc5050_arbitrum_nova_blockchain_support.py b/moonstreamdb/alembic/versions/2391c3cc5050_arbitrum_nova_blockchain_support.py new file mode 100644 index 00000000..c0f09a4f --- /dev/null +++ b/moonstreamdb/alembic/versions/2391c3cc5050_arbitrum_nova_blockchain_support.py @@ -0,0 +1,118 @@ +"""Arbitrum Nova blockchain support + +Revision ID: 2391c3cc5050 +Revises: 0f8ee1ebb45f +Create Date: 2024-01-30 11:55:24.251107 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '2391c3cc5050' +down_revision = '0f8ee1ebb45f' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('arbitrum_nova_blocks', + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('difficulty', sa.BigInteger(), nullable=True), + sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True), + sa.Column('gas_limit', sa.BigInteger(), nullable=True), + sa.Column('gas_used', sa.BigInteger(), nullable=True), + sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True), + sa.Column('miner', sa.VARCHAR(length=256), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('uncles', sa.VARCHAR(length=256), nullable=True), + sa.Column('size', sa.Integer(), nullable=True), + sa.Column('state_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('timestamp', sa.BigInteger(), nullable=True), + sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True), + sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.Column('sha3_uncles', sa.VARCHAR(length=256), nullable=True), + sa.Column('l1_block_number', sa.BigInteger(), nullable=True), + sa.Column('send_count', sa.BigInteger(), nullable=True), + sa.Column('send_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('mix_hash', sa.VARCHAR(length=256), nullable=True), + sa.PrimaryKeyConstraint('block_number', name=op.f('pk_arbitrum_nova_blocks')) + ) + op.create_index(op.f('ix_arbitrum_nova_blocks_block_number'), 'arbitrum_nova_blocks', ['block_number'], unique=True) + op.create_index(op.f('ix_arbitrum_nova_blocks_hash'), 'arbitrum_nova_blocks', ['hash'], unique=False) + op.create_index(op.f('ix_arbitrum_nova_blocks_timestamp'), 'arbitrum_nova_blocks', ['timestamp'], unique=False) + op.create_table('arbitrum_nova_labels', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('label', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=True), + sa.Column('address', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('block_timestamp', sa.BigInteger(), nullable=True), + sa.Column('log_index', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_arbitrum_nova_labels')), + sa.UniqueConstraint('id', name=op.f('uq_arbitrum_nova_labels_id')) + ) + op.create_index(op.f('ix_arbitrum_nova_labels_address'), 'arbitrum_nova_labels', ['address'], unique=False) + op.create_index(op.f('ix_arbitrum_nova_labels_block_number'), 'arbitrum_nova_labels', ['block_number'], unique=False) + op.create_index(op.f('ix_arbitrum_nova_labels_block_timestamp'), 'arbitrum_nova_labels', ['block_timestamp'], unique=False) + op.create_index(op.f('ix_arbitrum_nova_labels_label'), 'arbitrum_nova_labels', ['label'], unique=False) + op.create_index(op.f('ix_arbitrum_nova_labels_transaction_hash'), 'arbitrum_nova_labels', ['transaction_hash'], unique=False) + op.create_table('arbitrum_nova_transactions', + sa.Column('hash', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('from_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('to_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('input', sa.Text(), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_index', sa.BigInteger(), nullable=True), + sa.Column('transaction_type', sa.Integer(), nullable=True), + sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.Column('y_parity', sa.BigInteger(), nullable=True), + sa.ForeignKeyConstraint(['block_number'], ['arbitrum_nova_blocks.block_number'], name=op.f('fk_arbitrum_nova_transactions_block_number_arbitrum_nova_blocks'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('hash', name=op.f('pk_arbitrum_nova_transactions')) + ) + op.create_index(op.f('ix_arbitrum_nova_transactions_block_number'), 'arbitrum_nova_transactions', ['block_number'], unique=False) + op.create_index(op.f('ix_arbitrum_nova_transactions_from_address'), 'arbitrum_nova_transactions', ['from_address'], unique=False) + op.create_index(op.f('ix_arbitrum_nova_transactions_gas'), 'arbitrum_nova_transactions', ['gas'], unique=False) + op.create_index(op.f('ix_arbitrum_nova_transactions_gas_price'), 'arbitrum_nova_transactions', ['gas_price'], unique=False) + op.create_index(op.f('ix_arbitrum_nova_transactions_hash'), 'arbitrum_nova_transactions', ['hash'], unique=True) + op.create_index(op.f('ix_arbitrum_nova_transactions_to_address'), 'arbitrum_nova_transactions', ['to_address'], unique=False) + op.create_index(op.f('ix_arbitrum_nova_transactions_value'), 'arbitrum_nova_transactions', ['value'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_arbitrum_nova_transactions_value'), table_name='arbitrum_nova_transactions') + op.drop_index(op.f('ix_arbitrum_nova_transactions_to_address'), table_name='arbitrum_nova_transactions') + op.drop_index(op.f('ix_arbitrum_nova_transactions_hash'), table_name='arbitrum_nova_transactions') + op.drop_index(op.f('ix_arbitrum_nova_transactions_gas_price'), table_name='arbitrum_nova_transactions') + op.drop_index(op.f('ix_arbitrum_nova_transactions_gas'), table_name='arbitrum_nova_transactions') + op.drop_index(op.f('ix_arbitrum_nova_transactions_from_address'), table_name='arbitrum_nova_transactions') + op.drop_index(op.f('ix_arbitrum_nova_transactions_block_number'), table_name='arbitrum_nova_transactions') + op.drop_table('arbitrum_nova_transactions') + op.drop_index(op.f('ix_arbitrum_nova_labels_transaction_hash'), table_name='arbitrum_nova_labels') + op.drop_index(op.f('ix_arbitrum_nova_labels_label'), table_name='arbitrum_nova_labels') + op.drop_index(op.f('ix_arbitrum_nova_labels_block_timestamp'), table_name='arbitrum_nova_labels') + op.drop_index(op.f('ix_arbitrum_nova_labels_block_number'), table_name='arbitrum_nova_labels') + op.drop_index(op.f('ix_arbitrum_nova_labels_address'), table_name='arbitrum_nova_labels') + op.drop_table('arbitrum_nova_labels') + op.drop_index(op.f('ix_arbitrum_nova_blocks_timestamp'), table_name='arbitrum_nova_blocks') + op.drop_index(op.f('ix_arbitrum_nova_blocks_hash'), table_name='arbitrum_nova_blocks') + op.drop_index(op.f('ix_arbitrum_nova_blocks_block_number'), table_name='arbitrum_nova_blocks') + op.drop_table('arbitrum_nova_blocks') + # ### end Alembic commands ### diff --git a/moonstreamdb/moonstreamdb/blockchain.py b/moonstreamdb/moonstreamdb/blockchain.py index dcf636f6..66e8eb89 100644 --- a/moonstreamdb/moonstreamdb/blockchain.py +++ b/moonstreamdb/moonstreamdb/blockchain.py @@ -23,6 +23,9 @@ from .models import ( ZkSyncEraBlock, ZkSyncEraLabel, ZkSyncEraTransaction, + ArbitrumNovaBlock, + ArbitrumNovaTransaction, + ArbitrumNovaLabel, ) @@ -34,6 +37,7 @@ class AvailableBlockchainType(Enum): WYRM = "wyrm" ZKSYNC_ERA_TESTNET = "zksync_era_testnet" ZKSYNC_ERA = "zksync_era" + ARBITRUM_NOVA = "arbitrum_nova" def get_block_model( @@ -47,10 +51,11 @@ def get_block_model( WyrmBlock, ZkSyncEraTestnetBlock, ZkSyncEraBlock, + ArbitrumNovaBlock, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock set proper blocks model. """ block_model: Type[ @@ -62,6 +67,7 @@ def get_block_model( WyrmBlock, ZkSyncEraTestnetBlock, ZkSyncEraBlock, + ArbitrumNovaBlock, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -78,6 +84,8 @@ def get_block_model( block_model = ZkSyncEraTestnetBlock elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: block_model = ZkSyncEraBlock + elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: + block_model = ArbitrumNovaBlock else: raise Exception("Unsupported blockchain type provided") @@ -95,10 +103,11 @@ def get_label_model( WyrmLabel, ZkSyncEraTestnetLabel, ZkSyncEraLabel, + ArbitrumNovaLabel, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel set proper block label model. """ label_model: Type[ @@ -110,6 +119,7 @@ def get_label_model( WyrmLabel, ZkSyncEraTestnetLabel, ZkSyncEraLabel, + ArbitrumNovaLabel, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -126,6 +136,8 @@ def get_label_model( label_model = ZkSyncEraTestnetLabel elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: label_model = ZkSyncEraLabel + elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: + label_model = ArbitrumNovaLabel else: raise Exception("Unsupported blockchain type provided") @@ -143,10 +155,11 @@ def get_transaction_model( WyrmTransaction, ZkSyncEraTestnetTransaction, ZkSyncEraTransaction, + ArbitrumNovaTransaction, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction set proper block transactions model. """ transaction_model: Type[ @@ -158,6 +171,7 @@ def get_transaction_model( WyrmTransaction, ZkSyncEraTestnetTransaction, ZkSyncEraTransaction, + ArbitrumNovaTransaction, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -174,6 +188,8 @@ def get_transaction_model( transaction_model = ZkSyncEraTestnetTransaction elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: transaction_model = ZkSyncEraTransaction + elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: + transaction_model = ArbitrumNovaTransaction else: raise Exception("Unsupported blockchain type provided") diff --git a/moonstreamdb/moonstreamdb/models.py b/moonstreamdb/moonstreamdb/models.py index 3d7c2537..6200078f 100644 --- a/moonstreamdb/moonstreamdb/models.py +++ b/moonstreamdb/moonstreamdb/models.py @@ -214,21 +214,6 @@ class PolygonTransaction(Base): # type: ignore class PolygonLabel(Base): # type: ignore - """ - Example of label_data: - { - "label": "ERC20", - "label_data": { - "name": "Uniswap", - "symbol": "UNI" - } - }, - { - "label": "Exchange" - "label_data": {...} - } - """ - __tablename__ = "polygon_labels" __table_args__ = ( @@ -445,21 +430,6 @@ class XDaiTransaction(Base): # type: ignore class XDaiLabel(Base): # type: ignore - """ - Example of label_data: - { - "label": "ERC20", - "label_data": { - "name": "Uniswap", - "symbol": "UNI" - } - }, - { - "label": "Exchange" - "label_data": {...} - } - """ - __tablename__ = "xdai_labels" id = Column( @@ -551,21 +521,6 @@ class WyrmTransaction(Base): # type: ignore class WyrmLabel(Base): # type: ignore - """ - Example of label_data: - { - "label": "ERC20", - "label_data": { - "name": "Uniswap", - "symbol": "UNI" - } - }, - { - "label": "Exchange" - "label_data": {...} - } - """ - __tablename__ = "wyrm_labels" __table_args__ = ( @@ -682,21 +637,6 @@ class ZkSyncEraTransaction(Base): # type: ignore class ZkSyncEraLabel(Base): # type: ignore - """ - Example of label_data: - { - "label": "ERC20", - "label_data": { - "name": "Uniswap", - "symbol": "UNI" - } - }, - { - "label": "Exchange" - "label_data": {...} - } - """ - __tablename__ = "zksync_era_labels" __table_args__ = ( @@ -813,21 +753,6 @@ class ZkSyncEraTestnetTransaction(Base): # type: ignore class ZkSyncEraTestnetLabel(Base): # type: ignore - """ - Example of label_data: - { - "label": "ERC20", - "label_data": { - "name": "Uniswap", - "symbol": "UNI" - } - }, - { - "label": "Exchange" - "label_data": {...} - } - """ - __tablename__ = "zksync_era_testnet_labels" __table_args__ = ( @@ -876,6 +801,106 @@ class ZkSyncEraTestnetLabel(Base): # type: ignore ) +class ArbitrumNovaBlock(Base): # type: ignore + __tablename__ = "arbitrum_nova_blocks" + + block_number = Column( + BigInteger, primary_key=True, unique=True, nullable=False, index=True + ) + difficulty = Column(BigInteger) + extra_data = Column(VARCHAR(128)) + gas_limit = Column(BigInteger) + gas_used = Column(BigInteger) + base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + hash = Column(VARCHAR(256), index=True) + logs_bloom = Column(VARCHAR(1024)) + miner = Column(VARCHAR(256)) + nonce = Column(VARCHAR(256)) + parent_hash = Column(VARCHAR(256)) + receipt_root = Column(VARCHAR(256)) + uncles = Column(VARCHAR(256)) + size = Column(Integer) + state_root = Column(VARCHAR(256)) + timestamp = Column(BigInteger, index=True) + total_difficulty = Column(VARCHAR(256)) + transactions_root = Column(VARCHAR(256)) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + sha3_uncles = Column(VARCHAR(256), nullable=True) + l1_block_number = Column(BigInteger, nullable=True) + send_count = Column(BigInteger, nullable=True) + send_root = Column(VARCHAR(256), nullable=True) + mix_hash = Column(VARCHAR(256), nullable=True) + + +class ArbitrumNovaTransaction(Base): # type: ignore + __tablename__ = "arbitrum_nova_transactions" + + hash = Column( + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True + ) + block_number = Column( + BigInteger, + ForeignKey("arbitrum_nova_blocks.block_number", ondelete="CASCADE"), + nullable=False, + index=True, + ) + from_address = Column(VARCHAR(256), index=True) + to_address = Column(VARCHAR(256), index=True) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) + max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + input = Column(Text) + nonce = Column(VARCHAR(256)) + transaction_index = Column(BigInteger) + transaction_type = Column(Integer, nullable=True) + value = Column(Numeric(precision=78, scale=0), index=True) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + y_parity = Column(BigInteger, nullable=True) + + +class ArbitrumNovaLabel(Base): # type: ignore + __tablename__ = "arbitrum_nova_labels" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + nullable=False, + ) + label = Column(VARCHAR(256), nullable=False, index=True) + block_number = Column( + BigInteger, + nullable=True, + index=True, + ) + address = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + transaction_hash = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + label_data = Column(JSONB, nullable=True) + block_timestamp = Column(BigInteger, index=True) + log_index = Column(Integer, nullable=True) + created_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + class ESDFunctionSignature(Base): # type: ignore """ Function signature from blockchain (Ethereum/Polygon) Signature Database. diff --git a/moonstreamdb/moonstreamdb/networks.py b/moonstreamdb/moonstreamdb/networks.py index cb69538e..ae4ee215 100644 --- a/moonstreamdb/moonstreamdb/networks.py +++ b/moonstreamdb/moonstreamdb/networks.py @@ -24,6 +24,9 @@ from .models import ( ZkSyncEraBlock, ZkSyncEraLabel, ZkSyncEraTransaction, + ArbitrumNovaBlock, + ArbitrumNovaTransaction, + ArbitrumNovaLabel, ) @@ -35,6 +38,7 @@ class Network(Enum): wyrm = "wyrm" zksync_era_testnet = "zksync_era_testnet" zksync_era = "zksync_era" + arbitrum_nova = "arbitrum_nova" tx_raw_types = Union[ @@ -45,6 +49,7 @@ tx_raw_types = Union[ XDaiTransaction, ZkSyncEraTestnetTransaction, ZkSyncEraTransaction, + ArbitrumNovaTransaction, ] MODELS: Dict[Network, Dict[str, Base]] = { @@ -83,4 +88,9 @@ MODELS: Dict[Network, Dict[str, Base]] = { "labels": ZkSyncEraLabel, "transactions": ZkSyncEraTransaction, }, + Network.arbitrum_nova: { + "blocks": ArbitrumNovaBlock, + "labels": ArbitrumNovaLabel, + "transactions": ArbitrumNovaTransaction, + }, } diff --git a/moonstreamdb/moonstreamdb/version.py b/moonstreamdb/moonstreamdb/version.py index 24dde1b2..4229e2eb 100644 --- a/moonstreamdb/moonstreamdb/version.py +++ b/moonstreamdb/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.3.5" +MOONSTREAMDB_VERSION = "0.3.6"