mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #1009 from moonstream-to/arb-nova-crawlers 

Arbitrum nova crawlers and model
pull/1012/head^2 moonstreamdb/v0.3.6
Sergei Sumarokov 2024-02-21 17:41:20 +03:00 zatwierdzone przez GitHub
rodzic d571ca24c7 5a24a517bb
commit 28dc39c71d
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: B5690EEEBB952194
22 zmienionych plików z 402 dodań i 98 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

11
crawlers/deploy/arbitrum-nova-missing.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Fill missing blocks at Arbitrum Nova database
After=network.target
[Service]
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_nova -n
CPUWeight=50
SyslogIdentifier=arbitrum-nova-missing

9
crawlers/deploy/arbitrum-nova-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at Arbitrum Nova database
[Timer]
OnBootSec=120s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

17
crawlers/deploy/arbitrum-nova-moonworm-crawler.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=Arbitrum Nova moonworm crawler
After=network.target
StartLimitIntervalSec=300
StartLimitBurst=3
[Service]
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_nova --confirmations 10 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=arbitrum-nova-moonworm-crawler
[Install]
WantedBy=multi-user.target

17
crawlers/deploy/arbitrum-nova-synchronize.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=Arbitrum Nova block with transactions synchronizer
StartLimitIntervalSec=300
StartLimitBurst=3
After=network.target
[Service]
Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_nova -c 10 -j 2
CPUWeight=90
SyslogIdentifier=arbitrum-nova-synchronize
[Install]
WantedBy=multi-user.target

32
crawlers/deploy/deploy.bash
Wyświetl plik

@ -122,6 +122,12 @@ ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="zksync-era-testnet-
ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-testnet-historical-crawl-events.service"
ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-testnet-historical-crawl-events.timer"
# Arbitrum Nova
ARBITRUM_NOVA_MISSING_SERVICE_FILE="arbitrum-nova-missing.service"
ARBITRUM_NOVA_MISSING_TIMER_FILE="arbitrum-nova-missing.timer"
ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-nova-moonworm-crawler.service"
ARBITRUM_NOVA_SYNCHRONIZE_SERVICE="arbitrum-nova-synchronize.service"
set -eu
echo
@ -584,3 +590,29 @@ cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "/home/ubuntu/.config/sys
cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}"
# Arbitrum Nova
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova block with transactions syncronizer service definition with ${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova missing service and timer with: ${ARBITRUM_NOVA_MISSING_SERVICE_FILE}, ${ARBITRUM_NOVA_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MISSING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_MISSING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova moonworm crawler service definition with ${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"

2
crawlers/deploy/monitoring-crawlers.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/
EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env
ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service
ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service
CPUWeight=90
SyslogIdentifier=monitoring-crawlers

22
crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -1,5 +1,5 @@
import logging
from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait
from concurrent.futures import Future, ThreadPoolExecutor, wait
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
from uuid import UUID
@ -21,6 +21,7 @@ from web3.types import BlockData
from .data import DateRange
from .db import yield_db_session, yield_db_session_ctx
from .settings import (
MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI,
MOONSTREAM_CRAWL_WORKERS,
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
@ -76,6 +77,8 @@ def connect(
web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI
else:
raise Exception("Wrong blockchain type provided for web3 URI")
@ -94,6 +97,15 @@ def connect(
return web3_client
def hex_to_int(hex_str: Optional[str] = None) -> Optional[int]:
if hex_str is None:
return None
elif hex_str.startswith("0x"):
return int(hex_str, 16)
else:
return int(hex_str)
def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) -> None:
"""
Add block if doesn't presented in database.
@ -145,6 +157,12 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
if block.get("l1BatchTimestamp") is not None
else None
)
if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
block_obj.sha3_uncles = block.get("sha3Uncles", "")
block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber"))
block_obj.send_count = hex_to_int(block.get("sendCount"))
block_obj.send_root = block.get("sendRoot", "")
block_obj.mix_hash = block.get("mixHash", "")
db_session.add(block_obj)
@ -188,6 +206,8 @@ def add_block_transactions(
if tx.get("l1BatchTxIndex") is not None
else None
)
if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
tx_obj.y_parity = hex_to_int(tx.get("yParity"))
db_session.add(tx_obj)

12
crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py
Wyświetl plik

@ -6,13 +6,17 @@ from typing import Dict, List, Optional, Tuple
from uuid import UUID
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.networks import Network
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
MoonstreamEthereumStateProvider,
)
from moonstreamdb.networks import Network
from sqlalchemy.orm.session import Session
from web3 import Web3
from ..settings import (
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
HISTORICAL_CRAWLER_STATUSES,
)
from .crawler import (
EventCrawlJob,
FunctionCallCrawlJob,
@ -29,10 +33,6 @@ from .crawler import (
from .db import add_events_to_session, add_function_calls_to_session, commit_session
from .event_crawler import _crawl_events
from .function_call_crawler import _crawl_functions
from ..settings import (
HISTORICAL_CRAWLER_STATUSES,
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -134,6 +134,8 @@ def continuous_crawler(
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

13
crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py
Wyświetl plik

@ -5,24 +5,24 @@ import time
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Any, Callable, Dict, List, Optional, cast, Union, Tuple
from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
from uuid import UUID
from bugout.data import BugoutSearchResult, BugoutJournalEntries
from bugout.data import BugoutJournalEntries, BugoutSearchResult
from eth_typing.evm import ChecksumAddress
from moonstreamdb.blockchain import AvailableBlockchainType
from web3.main import Web3
from moonworm.deployment import find_deployment_block # type: ignore
from web3.main import Web3
from ..blockchain import connect
from ..reporter import reporter
from ..settings import (
BUGOUT_REQUEST_TIMEOUT_SECONDS,
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
HISTORICAL_CRAWLER_STATUSES,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
bugout_client,
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
HISTORICAL_CRAWLER_STATUSES,
)
logging.basicConfig(level=logging.INFO)
@ -37,6 +37,7 @@ class SubscriptionTypes(Enum):
WYRM_BLOCKCHAIN = "wyrm_smartcontract"
ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract"
ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract"
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
@ -145,6 +146,8 @@ def blockchain_type_to_subscription_type(
return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
return SubscriptionTypes.ARBITRUM_NOVA_BLOCKCHAIN
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

4
crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py
Wyświetl plik

@ -2,6 +2,7 @@ import logging
from typing import List
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.networks import Network # type: ignore
from moonworm.crawler.function_call_crawler import ( # type: ignore
ContractFunctionCall,
FunctionCallCrawler,
@ -9,7 +10,6 @@ from moonworm.crawler.function_call_crawler import ( # type: ignore
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
MoonstreamEthereumStateProvider,
)
from moonstreamdb.networks import Network # type: ignore
from moonworm.watch import MockState # type: ignore
from sqlalchemy.orm import Session
from web3 import Web3
@ -72,6 +72,8 @@ def function_call_crawler(
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

6
crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py
Wyświetl plik

@ -5,10 +5,10 @@ from uuid import UUID
from eth_typing.evm import ChecksumAddress
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.networks import Network # type: ignore
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
MoonstreamEthereumStateProvider,
)
from moonstreamdb.networks import Network # type: ignore
from sqlalchemy.orm.session import Session
from web3 import Web3
@ -19,7 +19,7 @@ from .crawler import (
update_entries_status_and_progress,
)
from .db import add_events_to_session, add_function_calls_to_session, commit_session
from .event_crawler import _crawl_events, _autoscale_crawl_events
from .event_crawler import _autoscale_crawl_events, _crawl_events
from .function_call_crawler import _crawl_functions
logging.basicConfig(level=logging.INFO)
@ -65,6 +65,8 @@ def historical_crawler(
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
else:
raise Exception("Unsupported blockchain type provided")

8
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -122,6 +122,14 @@ MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI = os.environ.get(
if MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI == "":
raise Exception(
"MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI env variable is not set"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
try:

8
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py
Wyświetl plik

@ -8,7 +8,7 @@ import logging
import time
from datetime import datetime, timedelta
from enum import Enum
from typing import Any, Callable, cast, Dict, List, Optional, Union
from typing import Any, Callable, Dict, List, Optional, Union, cast
from uuid import UUID
import boto3 # type: ignore
@ -23,8 +23,9 @@ from moonstreamdb.blockchain import (
get_label_model,
get_transaction_model,
)
from sqlalchemy import and_, distinct, extract, func, text
from sqlalchemy import and_
from sqlalchemy import cast as sqlalchemy_cast
from sqlalchemy import distinct, extract, func, text
from sqlalchemy.orm import Session
from sqlalchemy.sql.operators import in_op
from web3 import Web3
@ -54,6 +55,7 @@ subscription_id_by_blockchain = {
"wyrm": "wyrm_smartcontract",
"zksync_era_testnet": "zksync_era_testnet_smartcontract",
"zksync_era": "zksync_era_smartcontract",
"arbitrum_nova": "arbitrum_nova_smartcontract",
}
blockchain_by_subscription_id = {
@ -63,6 +65,7 @@ blockchain_by_subscription_id = {
"xdai_blockchain": "xdai",
"wyrm_blockchain": "wyrm",
"zksync_era_testnet_blockchain": "zksync_era_testnet",
"arbitrum_nova_blockchain": "arbitrum_nova",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"mumbai_smartcontract": "mumbai",
@ -70,6 +73,7 @@ blockchain_by_subscription_id = {
"wyrm_smartcontract": "wyrm",
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
"zksync_era_smartcontract": "zksync_era",
"arbitrum_nova_smartcontract": "arbitrum_nova",
}

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.3.5"
MOONCRAWL_VERSION = "0.3.6"

2
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -27,6 +27,8 @@ export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_mumb
export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_node>"
export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_testnet_node>"
export MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_node>"
export MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_arbitrum_nova_node>"
export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
# AWS environment variables

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -37,7 +37,7 @@ setup(
"bugout>=0.2.13",
"chardet",
"fastapi",
"moonstreamdb>=0.3.5",
"moonstreamdb>=0.3.6",
"moonstream>=0.1.1",
"moonworm[moonstream]>=0.6.2",
"humbug",

6
moonstreamdb/alembic/env.py
Wyświetl plik

@ -46,6 +46,9 @@ from moonstreamdb.models import (
ZkSyncEraTestnetBlock,
ZkSyncEraTestnetLabel,
ZkSyncEraTestnetTransaction,
ArbitrumNovaBlock,
ArbitrumNovaTransaction,
ArbitrumNovaLabel,
)
@ -72,6 +75,9 @@ def include_symbol(tablename, schema):
ZkSyncEraTestnetBlock.__tablename__,
ZkSyncEraTestnetLabel.__tablename__,
ZkSyncEraTestnetTransaction.__tablename__,
ArbitrumNovaBlock.__tablename__,
ArbitrumNovaTransaction.__tablename__,
ArbitrumNovaLabel.__tablename__,
}

118
moonstreamdb/alembic/versions/2391c3cc5050_arbitrum_nova_blockchain_support.py 100644
Wyświetl plik

@ -0,0 +1,118 @@
"""Arbitrum Nova blockchain support
Revision ID: 2391c3cc5050
Revises: 0f8ee1ebb45f
Create Date: 2024-01-30 11:55:24.251107
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '2391c3cc5050'
down_revision = '0f8ee1ebb45f'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('arbitrum_nova_blocks',
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('difficulty', sa.BigInteger(), nullable=True),
sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True),
sa.Column('gas_limit', sa.BigInteger(), nullable=True),
sa.Column('gas_used', sa.BigInteger(), nullable=True),
sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True),
sa.Column('miner', sa.VARCHAR(length=256), nullable=True),
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('uncles', sa.VARCHAR(length=256), nullable=True),
sa.Column('size', sa.Integer(), nullable=True),
sa.Column('state_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('timestamp', sa.BigInteger(), nullable=True),
sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True),
sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.Column('sha3_uncles', sa.VARCHAR(length=256), nullable=True),
sa.Column('l1_block_number', sa.BigInteger(), nullable=True),
sa.Column('send_count', sa.BigInteger(), nullable=True),
sa.Column('send_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('mix_hash', sa.VARCHAR(length=256), nullable=True),
sa.PrimaryKeyConstraint('block_number', name=op.f('pk_arbitrum_nova_blocks'))
)
op.create_index(op.f('ix_arbitrum_nova_blocks_block_number'), 'arbitrum_nova_blocks', ['block_number'], unique=True)
op.create_index(op.f('ix_arbitrum_nova_blocks_hash'), 'arbitrum_nova_blocks', ['hash'], unique=False)
op.create_index(op.f('ix_arbitrum_nova_blocks_timestamp'), 'arbitrum_nova_blocks', ['timestamp'], unique=False)
op.create_table('arbitrum_nova_labels',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('label', sa.VARCHAR(length=256), nullable=False),
sa.Column('block_number', sa.BigInteger(), nullable=True),
sa.Column('address', sa.VARCHAR(length=256), nullable=True),
sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column('block_timestamp', sa.BigInteger(), nullable=True),
sa.Column('log_index', sa.Integer(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_arbitrum_nova_labels')),
sa.UniqueConstraint('id', name=op.f('uq_arbitrum_nova_labels_id'))
)
op.create_index(op.f('ix_arbitrum_nova_labels_address'), 'arbitrum_nova_labels', ['address'], unique=False)
op.create_index(op.f('ix_arbitrum_nova_labels_block_number'), 'arbitrum_nova_labels', ['block_number'], unique=False)
op.create_index(op.f('ix_arbitrum_nova_labels_block_timestamp'), 'arbitrum_nova_labels', ['block_timestamp'], unique=False)
op.create_index(op.f('ix_arbitrum_nova_labels_label'), 'arbitrum_nova_labels', ['label'], unique=False)
op.create_index(op.f('ix_arbitrum_nova_labels_transaction_hash'), 'arbitrum_nova_labels', ['transaction_hash'], unique=False)
op.create_table('arbitrum_nova_transactions',
sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('input', sa.Text(), nullable=True),
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
sa.Column('transaction_index', sa.BigInteger(), nullable=True),
sa.Column('transaction_type', sa.Integer(), nullable=True),
sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.Column('y_parity', sa.BigInteger(), nullable=True),
sa.ForeignKeyConstraint(['block_number'], ['arbitrum_nova_blocks.block_number'], name=op.f('fk_arbitrum_nova_transactions_block_number_arbitrum_nova_blocks'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('hash', name=op.f('pk_arbitrum_nova_transactions'))
)
op.create_index(op.f('ix_arbitrum_nova_transactions_block_number'), 'arbitrum_nova_transactions', ['block_number'], unique=False)
op.create_index(op.f('ix_arbitrum_nova_transactions_from_address'), 'arbitrum_nova_transactions', ['from_address'], unique=False)
op.create_index(op.f('ix_arbitrum_nova_transactions_gas'), 'arbitrum_nova_transactions', ['gas'], unique=False)
op.create_index(op.f('ix_arbitrum_nova_transactions_gas_price'), 'arbitrum_nova_transactions', ['gas_price'], unique=False)
op.create_index(op.f('ix_arbitrum_nova_transactions_hash'), 'arbitrum_nova_transactions', ['hash'], unique=True)
op.create_index(op.f('ix_arbitrum_nova_transactions_to_address'), 'arbitrum_nova_transactions', ['to_address'], unique=False)
op.create_index(op.f('ix_arbitrum_nova_transactions_value'), 'arbitrum_nova_transactions', ['value'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_arbitrum_nova_transactions_value'), table_name='arbitrum_nova_transactions')
op.drop_index(op.f('ix_arbitrum_nova_transactions_to_address'), table_name='arbitrum_nova_transactions')
op.drop_index(op.f('ix_arbitrum_nova_transactions_hash'), table_name='arbitrum_nova_transactions')
op.drop_index(op.f('ix_arbitrum_nova_transactions_gas_price'), table_name='arbitrum_nova_transactions')
op.drop_index(op.f('ix_arbitrum_nova_transactions_gas'), table_name='arbitrum_nova_transactions')
op.drop_index(op.f('ix_arbitrum_nova_transactions_from_address'), table_name='arbitrum_nova_transactions')
op.drop_index(op.f('ix_arbitrum_nova_transactions_block_number'), table_name='arbitrum_nova_transactions')
op.drop_table('arbitrum_nova_transactions')
op.drop_index(op.f('ix_arbitrum_nova_labels_transaction_hash'), table_name='arbitrum_nova_labels')
op.drop_index(op.f('ix_arbitrum_nova_labels_label'), table_name='arbitrum_nova_labels')
op.drop_index(op.f('ix_arbitrum_nova_labels_block_timestamp'), table_name='arbitrum_nova_labels')
op.drop_index(op.f('ix_arbitrum_nova_labels_block_number'), table_name='arbitrum_nova_labels')
op.drop_index(op.f('ix_arbitrum_nova_labels_address'), table_name='arbitrum_nova_labels')
op.drop_table('arbitrum_nova_labels')
op.drop_index(op.f('ix_arbitrum_nova_blocks_timestamp'), table_name='arbitrum_nova_blocks')
op.drop_index(op.f('ix_arbitrum_nova_blocks_hash'), table_name='arbitrum_nova_blocks')
op.drop_index(op.f('ix_arbitrum_nova_blocks_block_number'), table_name='arbitrum_nova_blocks')
op.drop_table('arbitrum_nova_blocks')
# ### end Alembic commands ###

22
moonstreamdb/moonstreamdb/blockchain.py
Wyświetl plik

@ -23,6 +23,9 @@ from .models import (
ZkSyncEraBlock,
ZkSyncEraLabel,
ZkSyncEraTransaction,
ArbitrumNovaBlock,
ArbitrumNovaTransaction,
ArbitrumNovaLabel,
)
@ -34,6 +37,7 @@ class AvailableBlockchainType(Enum):
WYRM = "wyrm"
ZKSYNC_ERA_TESTNET = "zksync_era_testnet"
ZKSYNC_ERA = "zksync_era"
ARBITRUM_NOVA = "arbitrum_nova"
def get_block_model(
@ -47,10 +51,11 @@ def get_block_model(
WyrmBlock,
ZkSyncEraTestnetBlock,
ZkSyncEraBlock,
ArbitrumNovaBlock,
]
]:
"""
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock
set proper blocks model.
"""
block_model: Type[
@ -62,6 +67,7 @@ def get_block_model(
WyrmBlock,
ZkSyncEraTestnetBlock,
ZkSyncEraBlock,
ArbitrumNovaBlock,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
@ -78,6 +84,8 @@ def get_block_model(
block_model = ZkSyncEraTestnetBlock
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
block_model = ZkSyncEraBlock
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
block_model = ArbitrumNovaBlock
else:
raise Exception("Unsupported blockchain type provided")
@ -95,10 +103,11 @@ def get_label_model(
WyrmLabel,
ZkSyncEraTestnetLabel,
ZkSyncEraLabel,
ArbitrumNovaLabel,
]
]:
"""
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel
set proper block label model.
"""
label_model: Type[
@ -110,6 +119,7 @@ def get_label_model(
WyrmLabel,
ZkSyncEraTestnetLabel,
ZkSyncEraLabel,
ArbitrumNovaLabel,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
@ -126,6 +136,8 @@ def get_label_model(
label_model = ZkSyncEraTestnetLabel
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
label_model = ZkSyncEraLabel
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
label_model = ArbitrumNovaLabel
else:
raise Exception("Unsupported blockchain type provided")
@ -143,10 +155,11 @@ def get_transaction_model(
WyrmTransaction,
ZkSyncEraTestnetTransaction,
ZkSyncEraTransaction,
ArbitrumNovaTransaction,
]
]:
"""
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction
set proper block transactions model.
"""
transaction_model: Type[
@ -158,6 +171,7 @@ def get_transaction_model(
WyrmTransaction,
ZkSyncEraTestnetTransaction,
ZkSyncEraTransaction,
ArbitrumNovaTransaction,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
@ -174,6 +188,8 @@ def get_transaction_model(
transaction_model = ZkSyncEraTestnetTransaction
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
transaction_model = ZkSyncEraTransaction
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
transaction_model = ArbitrumNovaTransaction
else:
raise Exception("Unsupported blockchain type provided")

175
moonstreamdb/moonstreamdb/models.py
Wyświetl plik

@ -214,21 +214,6 @@ class PolygonTransaction(Base): # type: ignore
class PolygonLabel(Base): # type: ignore
"""
Example of label_data:
{
"label": "ERC20",
"label_data": {
"name": "Uniswap",
"symbol": "UNI"
}
},
{
"label": "Exchange"
"label_data": {...}
}
"""
__tablename__ = "polygon_labels"
__table_args__ = (
@ -445,21 +430,6 @@ class XDaiTransaction(Base): # type: ignore
class XDaiLabel(Base): # type: ignore
"""
Example of label_data:
{
"label": "ERC20",
"label_data": {
"name": "Uniswap",
"symbol": "UNI"
}
},
{
"label": "Exchange"
"label_data": {...}
}
"""
__tablename__ = "xdai_labels"
id = Column(
@ -551,21 +521,6 @@ class WyrmTransaction(Base): # type: ignore
class WyrmLabel(Base): # type: ignore
"""
Example of label_data:
{
"label": "ERC20",
"label_data": {
"name": "Uniswap",
"symbol": "UNI"
}
},
{
"label": "Exchange"
"label_data": {...}
}
"""
__tablename__ = "wyrm_labels"
__table_args__ = (
@ -682,21 +637,6 @@ class ZkSyncEraTransaction(Base): # type: ignore
class ZkSyncEraLabel(Base): # type: ignore
"""
Example of label_data:
{
"label": "ERC20",
"label_data": {
"name": "Uniswap",
"symbol": "UNI"
}
},
{
"label": "Exchange"
"label_data": {...}
}
"""
__tablename__ = "zksync_era_labels"
__table_args__ = (
@ -813,21 +753,6 @@ class ZkSyncEraTestnetTransaction(Base): # type: ignore
class ZkSyncEraTestnetLabel(Base): # type: ignore
"""
Example of label_data:
{
"label": "ERC20",
"label_data": {
"name": "Uniswap",
"symbol": "UNI"
}
},
{
"label": "Exchange"
"label_data": {...}
}
"""
__tablename__ = "zksync_era_testnet_labels"
__table_args__ = (
@ -876,6 +801,106 @@ class ZkSyncEraTestnetLabel(Base): # type: ignore
)
class ArbitrumNovaBlock(Base): # type: ignore
__tablename__ = "arbitrum_nova_blocks"
block_number = Column(
BigInteger, primary_key=True, unique=True, nullable=False, index=True
)
difficulty = Column(BigInteger)
extra_data = Column(VARCHAR(128))
gas_limit = Column(BigInteger)
gas_used = Column(BigInteger)
base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
hash = Column(VARCHAR(256), index=True)
logs_bloom = Column(VARCHAR(1024))
miner = Column(VARCHAR(256))
nonce = Column(VARCHAR(256))
parent_hash = Column(VARCHAR(256))
receipt_root = Column(VARCHAR(256))
uncles = Column(VARCHAR(256))
size = Column(Integer)
state_root = Column(VARCHAR(256))
timestamp = Column(BigInteger, index=True)
total_difficulty = Column(VARCHAR(256))
transactions_root = Column(VARCHAR(256))
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
sha3_uncles = Column(VARCHAR(256), nullable=True)
l1_block_number = Column(BigInteger, nullable=True)
send_count = Column(BigInteger, nullable=True)
send_root = Column(VARCHAR(256), nullable=True)
mix_hash = Column(VARCHAR(256), nullable=True)
class ArbitrumNovaTransaction(Base): # type: ignore
__tablename__ = "arbitrum_nova_transactions"
hash = Column(
VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
)
block_number = Column(
BigInteger,
ForeignKey("arbitrum_nova_blocks.block_number", ondelete="CASCADE"),
nullable=False,
index=True,
)
from_address = Column(VARCHAR(256), index=True)
to_address = Column(VARCHAR(256), index=True)
gas = Column(Numeric(precision=78, scale=0), index=True)
gas_price = Column(Numeric(precision=78, scale=0), index=True)
max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
input = Column(Text)
nonce = Column(VARCHAR(256))
transaction_index = Column(BigInteger)
transaction_type = Column(Integer, nullable=True)
value = Column(Numeric(precision=78, scale=0), index=True)
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
y_parity = Column(BigInteger, nullable=True)
class ArbitrumNovaLabel(Base): # type: ignore
__tablename__ = "arbitrum_nova_labels"
id = Column(
UUID(as_uuid=True),
primary_key=True,
default=uuid.uuid4,
unique=True,
nullable=False,
)
label = Column(VARCHAR(256), nullable=False, index=True)
block_number = Column(
BigInteger,
nullable=True,
index=True,
)
address = Column(
VARCHAR(256),
nullable=True,
index=True,
)
transaction_hash = Column(
VARCHAR(256),
nullable=True,
index=True,
)
label_data = Column(JSONB, nullable=True)
block_timestamp = Column(BigInteger, index=True)
log_index = Column(Integer, nullable=True)
created_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
class ESDFunctionSignature(Base): # type: ignore
"""
Function signature from blockchain (Ethereum/Polygon) Signature Database.

10
moonstreamdb/moonstreamdb/networks.py
Wyświetl plik

@ -24,6 +24,9 @@ from .models import (
ZkSyncEraBlock,
ZkSyncEraLabel,
ZkSyncEraTransaction,
ArbitrumNovaBlock,
ArbitrumNovaTransaction,
ArbitrumNovaLabel,
)
@ -35,6 +38,7 @@ class Network(Enum):
wyrm = "wyrm"
zksync_era_testnet = "zksync_era_testnet"
zksync_era = "zksync_era"
arbitrum_nova = "arbitrum_nova"
tx_raw_types = Union[
@ -45,6 +49,7 @@ tx_raw_types = Union[
XDaiTransaction,
ZkSyncEraTestnetTransaction,
ZkSyncEraTransaction,
ArbitrumNovaTransaction,
]
MODELS: Dict[Network, Dict[str, Base]] = {
@ -83,4 +88,9 @@ MODELS: Dict[Network, Dict[str, Base]] = {
"labels": ZkSyncEraLabel,
"transactions": ZkSyncEraTransaction,
},
Network.arbitrum_nova: {
"blocks": ArbitrumNovaBlock,
"labels": ArbitrumNovaLabel,
"transactions": ArbitrumNovaTransaction,
},
}

2
moonstreamdb/moonstreamdb/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream database version.
"""
MOONSTREAMDB_VERSION = "0.3.5"
MOONSTREAMDB_VERSION = "0.3.6"