mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #1012 from moonstream-to/arb-sepolia-crawlers 

Arb sepolia crawlers
pull/1016/head^2 moonstreamdb/v0.3.7
Andrey Dolgolev 2024-02-21 16:57:55 +02:00 zatwierdzone przez GitHub
rodzic 28dc39c71d 64a4036fd7
commit 192961d008
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: B5690EEEBB952194
22 zmienionych plików z 551 dodań i 28 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

11
crawlers/deploy/arbitrum-sepolia-missing.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Fill missing blocks at Arbitrum Sepolia database
After=network.target
[Service]
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_sepolia -n
CPUWeight=50
SyslogIdentifier=arbitrum-sepolia-missing

9
crawlers/deploy/arbitrum-sepolia-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at Arbitrum Sepolia database
[Timer]
OnBootSec=120s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

17
crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=Arbitrum Sepolia moonworm crawler
After=network.target
StartLimitIntervalSec=300
StartLimitBurst=3
[Service]
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_sepolia --confirmations 10 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=arbitrum-sepolia-moonworm-crawler
[Install]
WantedBy=multi-user.target

17
crawlers/deploy/arbitrum-sepolia-synchronize.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=Arbitrum Sepolia block with transactions synchronizer
StartLimitIntervalSec=300
StartLimitBurst=3
After=network.target
[Service]
Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_sepolia -c 10 -j 2
CPUWeight=90
SyslogIdentifier=arbitrum-sepolia-synchronize
[Install]
WantedBy=multi-user.target

34
crawlers/deploy/deploy.bash
Wyświetl plik

@ -128,6 +128,12 @@ ARBITRUM_NOVA_MISSING_TIMER_FILE="arbitrum-nova-missing.timer"
ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-nova-moonworm-crawler.service"
ARBITRUM_NOVA_SYNCHRONIZE_SERVICE="arbitrum-nova-synchronize.service"
# Arbitrum Sepolia
ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE="arbitrum-sepolia-missing.service"
ARBITRUM_SEPOLIA_MISSING_TIMER_FILE="arbitrum-sepolia-missing.timer"
ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-sepolia-moonworm-crawler.service"
ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE="arbitrum-sepolia-synchronize.service"
set -eu
echo
@ -616,3 +622,31 @@ chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"
# Arbitrum Sepolia
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Arbitrum Sepolia block with transactions syncronizer service definition with ${ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Arbitrum Sepolia missing service and timer with: ${ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE}, ${ARBITRUM_SEPOLIA_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_MISSING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_MISSING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Arbitrum Sepolia moonworm crawler service definition with ${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"

2
crawlers/deploy/monitoring-crawlers.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/
EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env
ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service
ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service
CPUWeight=90
SyslogIdentifier=monitoring-crawlers

21
crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -22,6 +22,7 @@ from .data import DateRange
from .db import yield_db_session, yield_db_session_ctx
from .settings import (
MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI,
MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI,
MOONSTREAM_CRAWL_WORKERS,
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
@ -79,6 +80,8 @@ def connect(
web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
web3_uri = MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI
else:
raise Exception("Wrong blockchain type provided for web3 URI")
@ -120,9 +123,11 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
block_obj = block_model(
block_number=block.number,
difficulty=block.difficulty,
extra_data=None
if block.get("extraData", None) is None
else block.get("extraData").hex(),
extra_data=(
None
if block.get("extraData", None) is None
else block.get("extraData").hex()
),
gas_limit=block.gasLimit,
gas_used=block.gasUsed,
base_fee_per_gas=block.get("baseFeePerGas", None),
@ -164,6 +169,13 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
block_obj.send_root = block.get("sendRoot", "")
block_obj.mix_hash = block.get("mixHash", "")
if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
block_obj.sha3_uncles = block.get("sha3Uncles", "")
block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber"))
block_obj.send_count = hex_to_int(block.get("sendCount"))
block_obj.send_root = block.get("sendRoot", "")
block_obj.mix_hash = block.get("mixHash", "")
db_session.add(block_obj)
@ -209,6 +221,9 @@ def add_block_transactions(
if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
tx_obj.y_parity = hex_to_int(tx.get("yParity"))
if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
tx_obj.y_parity = hex_to_int(tx.get("yParity"))
db_session.add(tx_obj)

15
crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py
Wyświetl plik

@ -136,6 +136,9 @@ def continuous_crawler(
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
network = Network.arbitrum_sepolia
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
@ -259,9 +262,9 @@ def continuous_crawler(
heartbeat_template["current_function_call_jobs_length"] = len(
function_call_crawl_jobs
)
heartbeat_template[
"function_call metrics"
] = ethereum_state_provider.metrics
heartbeat_template["function_call metrics"] = (
ethereum_state_provider.metrics
)
heartbeat(
crawler_type=crawler_type,
blockchain_type=blockchain_type,
@ -303,9 +306,9 @@ def continuous_crawler(
)
),
)
heartbeat_template[
"die_reason"
] = f"{e.__class__.__name__}: {e}\n error_summary: {error_summary}\n error_traceback: {error_traceback}"
heartbeat_template["die_reason"] = (
f"{e.__class__.__name__}: {e}\n error_summary: {error_summary}\n error_traceback: {error_traceback}"
)
heartbeat_template["last_block"] = end_block
heartbeat(
crawler_type=crawler_type,

9
crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py
Wyświetl plik

@ -38,6 +38,7 @@ class SubscriptionTypes(Enum):
ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract"
ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract"
ARBITRUM_SEPOLIA_BLOCKCHAIN = "arbitrum_sepolia_smartcontract"
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
@ -148,6 +149,8 @@ def blockchain_type_to_subscription_type(
return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
return SubscriptionTypes.ARBITRUM_NOVA_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
return SubscriptionTypes.ARBITRUM_SEPOLIA_BLOCKCHAIN
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
@ -554,9 +557,9 @@ def update_job_tags(
for contract_address, entries_ids in event.address_entries.items():
for entry_id in entries_ids.keys():
if entry_id in entry_tags_by_id:
event.address_entries[contract_address][
entry_id
] = entry_tags_by_id[entry_id]
event.address_entries[contract_address][entry_id] = (
entry_tags_by_id[entry_id]
)
if isinstance(event, FunctionCallCrawlJob):
for entry_id in event.entries_tags.keys():

2
crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py
Wyświetl plik

@ -74,6 +74,8 @@ def function_call_crawler(
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
network = Network.arbitrum_sepolia
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

2
crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py
Wyświetl plik

@ -67,6 +67,8 @@ def historical_crawler(
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
network = Network.arbitrum_sepolia
else:
raise Exception("Unsupported blockchain type provided")

8
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -130,6 +130,14 @@ if MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI == "":
"MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI env variable is not set"
)
MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI == "":
raise Exception(
"MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI env variable is not set"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
try:

22
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py
Wyświetl plik

@ -1,6 +1,7 @@
"""
Generates dashboard.
"""
import argparse
import hashlib
import json
@ -56,6 +57,7 @@ subscription_id_by_blockchain = {
"zksync_era_testnet": "zksync_era_testnet_smartcontract",
"zksync_era": "zksync_era_smartcontract",
"arbitrum_nova": "arbitrum_nova_smartcontract",
"arbitrum_sepolia": "arbitrum_sepolia_smartcontract",
}
blockchain_by_subscription_id = {
@ -66,6 +68,7 @@ blockchain_by_subscription_id = {
"wyrm_blockchain": "wyrm",
"zksync_era_testnet_blockchain": "zksync_era_testnet",
"arbitrum_nova_blockchain": "arbitrum_nova",
"arbitrum_sepolia_blockchain": "arbitrum_sepolia",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"mumbai_smartcontract": "mumbai",
@ -74,6 +77,7 @@ blockchain_by_subscription_id = {
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
"zksync_era_smartcontract": "zksync_era",
"arbitrum_nova_smartcontract": "arbitrum_nova",
"arbitrum_sepolia_smartcontract": "arbitrum_sepolia",
}
@ -931,9 +935,9 @@ def stats_generate_handler(args: argparse.Namespace):
s3_subscription_data_object: Dict[str, Any] = {}
s3_subscription_data_object[
"blocks_state"
] = s3_data_object_for_contract["blocks_state"]
s3_subscription_data_object["blocks_state"] = (
s3_data_object_for_contract["blocks_state"]
)
if dashboard_id in merged_external_calls:
for (
@ -952,9 +956,9 @@ def stats_generate_handler(args: argparse.Namespace):
}
)
s3_subscription_data_object[
"web3_metric"
] = extention_data
s3_subscription_data_object["web3_metric"] = (
extention_data
)
# list of user defined events
@ -966,9 +970,9 @@ def stats_generate_handler(args: argparse.Namespace):
for event in events_list:
if event in events_data:
s3_subscription_data_object["events"][
event
] = events_data[event]
s3_subscription_data_object["events"][event] = (
events_data[event]
)
# list of user defined functions

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.3.6"
MOONCRAWL_VERSION = "0.3.7"

1
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -29,6 +29,7 @@ export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_n
export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_testnet_node>"
export MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_node>"
export MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_arbitrum_nova_node>"
export MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_arbitrum_sepolia_node>"
export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
# AWS environment variables

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -37,7 +37,7 @@ setup(
"bugout>=0.2.13",
"chardet",
"fastapi",
"moonstreamdb>=0.3.6",
"moonstreamdb>=0.3.7",
"moonstream>=0.1.1",
"moonworm[moonstream]>=0.6.2",
"humbug",

6
moonstreamdb/alembic/env.py
Wyświetl plik

@ -49,6 +49,9 @@ from moonstreamdb.models import (
ArbitrumNovaBlock,
ArbitrumNovaTransaction,
ArbitrumNovaLabel,
ArbitrumSepoliaBlock,
ArbitrumSepoliaTransaction,
ArbitrumSepoliaLabel,
)
@ -78,6 +81,9 @@ def include_symbol(tablename, schema):
ArbitrumNovaBlock.__tablename__,
ArbitrumNovaTransaction.__tablename__,
ArbitrumNovaLabel.__tablename__,
ArbitrumSepoliaBlock.__tablename__,
ArbitrumSepoliaTransaction.__tablename__,
ArbitrumSepoliaLabel.__tablename__,
}

265
moonstreamdb/alembic/versions/b38b75b11247_arbitrum_sepolia_blockchain_support.py 100644
Wyświetl plik

@ -0,0 +1,265 @@
"""Arbitrum Sepolia blockchain support
Revision ID: b38b75b11247
Revises: 2391c3cc5050
Create Date: 2024-02-20 15:16:40.393985
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "b38b75b11247"
down_revision = "2391c3cc5050"
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"arbitrum_sepolia_blocks",
sa.Column("block_number", sa.BigInteger(), nullable=False),
sa.Column("difficulty", sa.BigInteger(), nullable=True),
sa.Column("extra_data", sa.VARCHAR(length=128), nullable=True),
sa.Column("gas_limit", sa.BigInteger(), nullable=True),
sa.Column("gas_used", sa.BigInteger(), nullable=True),
sa.Column("base_fee_per_gas", sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column("hash", sa.VARCHAR(length=256), nullable=True),
sa.Column("logs_bloom", sa.VARCHAR(length=1024), nullable=True),
sa.Column("miner", sa.VARCHAR(length=256), nullable=True),
sa.Column("nonce", sa.VARCHAR(length=256), nullable=True),
sa.Column("parent_hash", sa.VARCHAR(length=256), nullable=True),
sa.Column("receipt_root", sa.VARCHAR(length=256), nullable=True),
sa.Column("uncles", sa.VARCHAR(length=256), nullable=True),
sa.Column("size", sa.Integer(), nullable=True),
sa.Column("state_root", sa.VARCHAR(length=256), nullable=True),
sa.Column("timestamp", sa.BigInteger(), nullable=True),
sa.Column("total_difficulty", sa.VARCHAR(length=256), nullable=True),
sa.Column("transactions_root", sa.VARCHAR(length=256), nullable=True),
sa.Column(
"indexed_at",
sa.DateTime(timezone=True),
server_default=sa.text("TIMEZONE('utc', statement_timestamp())"),
nullable=False,
),
sa.Column("sha3_uncles", sa.VARCHAR(length=256), nullable=True),
sa.Column("l1_block_number", sa.BigInteger(), nullable=True),
sa.Column("send_count", sa.BigInteger(), nullable=True),
sa.Column("send_root", sa.VARCHAR(length=256), nullable=True),
sa.Column("mix_hash", sa.VARCHAR(length=256), nullable=True),
sa.PrimaryKeyConstraint(
"block_number", name=op.f("pk_arbitrum_sepolia_blocks")
),
)
op.create_index(
op.f("ix_arbitrum_sepolia_blocks_block_number"),
"arbitrum_sepolia_blocks",
["block_number"],
unique=True,
)
op.create_index(
op.f("ix_arbitrum_sepolia_blocks_hash"),
"arbitrum_sepolia_blocks",
["hash"],
unique=False,
)
op.create_index(
op.f("ix_arbitrum_sepolia_blocks_timestamp"),
"arbitrum_sepolia_blocks",
["timestamp"],
unique=False,
)
op.create_table(
"arbitrum_sepolia_labels",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("label", sa.VARCHAR(length=256), nullable=False),
sa.Column("block_number", sa.BigInteger(), nullable=True),
sa.Column("address", sa.VARCHAR(length=256), nullable=True),
sa.Column("transaction_hash", sa.VARCHAR(length=256), nullable=True),
sa.Column("label_data", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("block_timestamp", sa.BigInteger(), nullable=True),
sa.Column("log_index", sa.Integer(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("TIMEZONE('utc', statement_timestamp())"),
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_arbitrum_sepolia_labels")),
sa.UniqueConstraint("id", name=op.f("uq_arbitrum_sepolia_labels_id")),
)
op.create_index(
op.f("ix_arbitrum_sepolia_labels_address"),
"arbitrum_sepolia_labels",
["address"],
unique=False,
)
op.create_index(
op.f("ix_arbitrum_sepolia_labels_block_number"),
"arbitrum_sepolia_labels",
["block_number"],
unique=False,
)
op.create_index(
op.f("ix_arbitrum_sepolia_labels_block_timestamp"),
"arbitrum_sepolia_labels",
["block_timestamp"],
unique=False,
)
op.create_index(
op.f("ix_arbitrum_sepolia_labels_label"),
"arbitrum_sepolia_labels",
["label"],
unique=False,
)
op.create_index(
op.f("ix_arbitrum_sepolia_labels_transaction_hash"),
"arbitrum_sepolia_labels",
["transaction_hash"],
unique=False,
)
op.create_table(
"arbitrum_sepolia_transactions",
sa.Column("hash", sa.VARCHAR(length=256), nullable=False),
sa.Column("block_number", sa.BigInteger(), nullable=False),
sa.Column("from_address", sa.VARCHAR(length=256), nullable=True),
sa.Column("to_address", sa.VARCHAR(length=256), nullable=True),
sa.Column("gas", sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column("gas_price", sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column("max_fee_per_gas", sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column(
"max_priority_fee_per_gas", sa.Numeric(precision=78, scale=0), nullable=True
),
sa.Column("input", sa.Text(), nullable=True),
sa.Column("nonce", sa.VARCHAR(length=256), nullable=True),
sa.Column("transaction_index", sa.BigInteger(), nullable=True),
sa.Column("transaction_type", sa.Integer(), nullable=True),
sa.Column("value", sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column(
"indexed_at",
sa.DateTime(timezone=True),
server_default=sa.text("TIMEZONE('utc', statement_timestamp())"),
nullable=False,
),
sa.Column("y_parity", sa.BigInteger(), nullable=True),
sa.ForeignKeyConstraint(
["block_number"],
["arbitrum_sepolia_blocks.block_number"],
name=op.f(
"fk_arbitrum_sepolia_transactions_block_number_arbitrum_sepolia_blocks"
),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("hash", name=op.f("pk_arbitrum_sepolia_transactions")),
)
op.create_index(
op.f("ix_arbitrum_sepolia_transactions_block_number"),
"arbitrum_sepolia_transactions",
["block_number"],
unique=False,
)
op.create_index(
op.f("ix_arbitrum_sepolia_transactions_from_address"),
"arbitrum_sepolia_transactions",
["from_address"],
unique=False,
)
op.create_index(
op.f("ix_arbitrum_sepolia_transactions_gas"),
"arbitrum_sepolia_transactions",
["gas"],
unique=False,
)
op.create_index(
op.f("ix_arbitrum_sepolia_transactions_gas_price"),
"arbitrum_sepolia_transactions",
["gas_price"],
unique=False,
)
op.create_index(
op.f("ix_arbitrum_sepolia_transactions_hash"),
"arbitrum_sepolia_transactions",
["hash"],
unique=True,
)
op.create_index(
op.f("ix_arbitrum_sepolia_transactions_to_address"),
"arbitrum_sepolia_transactions",
["to_address"],
unique=False,
)
op.create_index(
op.f("ix_arbitrum_sepolia_transactions_value"),
"arbitrum_sepolia_transactions",
["value"],
unique=False,
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###)
op.drop_index(
op.f("ix_arbitrum_sepolia_transactions_value"),
table_name="arbitrum_sepolia_transactions",
)
op.drop_index(
op.f("ix_arbitrum_sepolia_transactions_to_address"),
table_name="arbitrum_sepolia_transactions",
)
op.drop_index(
op.f("ix_arbitrum_sepolia_transactions_hash"),
table_name="arbitrum_sepolia_transactions",
)
op.drop_index(
op.f("ix_arbitrum_sepolia_transactions_gas_price"),
table_name="arbitrum_sepolia_transactions",
)
op.drop_index(
op.f("ix_arbitrum_sepolia_transactions_gas"),
table_name="arbitrum_sepolia_transactions",
)
op.drop_index(
op.f("ix_arbitrum_sepolia_transactions_from_address"),
table_name="arbitrum_sepolia_transactions",
)
op.drop_index(
op.f("ix_arbitrum_sepolia_transactions_block_number"),
table_name="arbitrum_sepolia_transactions",
)
op.drop_table("arbitrum_sepolia_transactions")
op.drop_index(
op.f("ix_arbitrum_sepolia_labels_transaction_hash"),
table_name="arbitrum_sepolia_labels",
)
op.drop_index(
op.f("ix_arbitrum_sepolia_labels_label"), table_name="arbitrum_sepolia_labels"
)
op.drop_index(
op.f("ix_arbitrum_sepolia_labels_block_timestamp"),
table_name="arbitrum_sepolia_labels",
)
op.drop_index(
op.f("ix_arbitrum_sepolia_labels_block_number"),
table_name="arbitrum_sepolia_labels",
)
op.drop_index(
op.f("ix_arbitrum_sepolia_labels_address"), table_name="arbitrum_sepolia_labels"
)
op.drop_table("arbitrum_sepolia_labels")
op.drop_index(
op.f("ix_arbitrum_sepolia_blocks_timestamp"),
table_name="arbitrum_sepolia_blocks",
)
op.drop_index(
op.f("ix_arbitrum_sepolia_blocks_hash"), table_name="arbitrum_sepolia_blocks"
)
op.drop_index(
op.f("ix_arbitrum_sepolia_blocks_block_number"),
table_name="arbitrum_sepolia_blocks",
)
op.drop_table("arbitrum_sepolia_blocks")
# ### end Alembic commands ###

22
moonstreamdb/moonstreamdb/blockchain.py
Wyświetl plik

@ -26,6 +26,9 @@ from .models import (
ArbitrumNovaBlock,
ArbitrumNovaTransaction,
ArbitrumNovaLabel,
ArbitrumSepoliaBlock,
ArbitrumSepoliaTransaction,
ArbitrumSepoliaLabel,
)
@ -38,6 +41,7 @@ class AvailableBlockchainType(Enum):
ZKSYNC_ERA_TESTNET = "zksync_era_testnet"
ZKSYNC_ERA = "zksync_era"
ARBITRUM_NOVA = "arbitrum_nova"
ARBITRUM_SEPOLIA = "arbitrum_sepolia"
def get_block_model(
@ -52,10 +56,11 @@ def get_block_model(
ZkSyncEraTestnetBlock,
ZkSyncEraBlock,
ArbitrumNovaBlock,
ArbitrumSepoliaBlock,
]
]:
"""
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock, ArbitrumSepoliaBlock
set proper blocks model.
"""
block_model: Type[
@ -68,6 +73,7 @@ def get_block_model(
ZkSyncEraTestnetBlock,
ZkSyncEraBlock,
ArbitrumNovaBlock,
ArbitrumSepoliaBlock,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
@ -86,6 +92,8 @@ def get_block_model(
block_model = ZkSyncEraBlock
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
block_model = ArbitrumNovaBlock
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
block_model = ArbitrumSepoliaBlock
else:
raise Exception("Unsupported blockchain type provided")
@ -104,10 +112,11 @@ def get_label_model(
ZkSyncEraTestnetLabel,
ZkSyncEraLabel,
ArbitrumNovaLabel,
ArbitrumSepoliaLabel,
]
]:
"""
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel, ArbitrumSepoliaLabel
set proper block label model.
"""
label_model: Type[
@ -120,6 +129,7 @@ def get_label_model(
ZkSyncEraTestnetLabel,
ZkSyncEraLabel,
ArbitrumNovaLabel,
ArbitrumSepoliaLabel,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
@ -138,6 +148,8 @@ def get_label_model(
label_model = ZkSyncEraLabel
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
label_model = ArbitrumNovaLabel
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
label_model = ArbitrumSepoliaLabel
else:
raise Exception("Unsupported blockchain type provided")
@ -156,10 +168,11 @@ def get_transaction_model(
ZkSyncEraTestnetTransaction,
ZkSyncEraTransaction,
ArbitrumNovaTransaction,
ArbitrumSepoliaTransaction,
]
]:
"""
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction
set proper block transactions model.
"""
transaction_model: Type[
@ -172,6 +185,7 @@ def get_transaction_model(
ZkSyncEraTestnetTransaction,
ZkSyncEraTransaction,
ArbitrumNovaTransaction,
ArbitrumSepoliaTransaction,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
@ -190,6 +204,8 @@ def get_transaction_model(
transaction_model = ZkSyncEraTransaction
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
transaction_model = ArbitrumNovaTransaction
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
transaction_model = ArbitrumSepoliaTransaction
else:
raise Exception("Unsupported blockchain type provided")

100
moonstreamdb/moonstreamdb/models.py
Wyświetl plik

@ -901,6 +901,106 @@ class ArbitrumNovaLabel(Base): # type: ignore
)
class ArbitrumSepoliaBlock(Base): # type: ignore
__tablename__ = "arbitrum_sepolia_blocks"
block_number = Column(
BigInteger, primary_key=True, unique=True, nullable=False, index=True
)
difficulty = Column(BigInteger)
extra_data = Column(VARCHAR(128))
gas_limit = Column(BigInteger)
gas_used = Column(BigInteger)
base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
hash = Column(VARCHAR(256), index=True)
logs_bloom = Column(VARCHAR(1024))
miner = Column(VARCHAR(256))
nonce = Column(VARCHAR(256))
parent_hash = Column(VARCHAR(256))
receipt_root = Column(VARCHAR(256))
uncles = Column(VARCHAR(256))
size = Column(Integer)
state_root = Column(VARCHAR(256))
timestamp = Column(BigInteger, index=True)
total_difficulty = Column(VARCHAR(256))
transactions_root = Column(VARCHAR(256))
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
sha3_uncles = Column(VARCHAR(256), nullable=True)
l1_block_number = Column(BigInteger, nullable=True)
send_count = Column(BigInteger, nullable=True)
send_root = Column(VARCHAR(256), nullable=True)
mix_hash = Column(VARCHAR(256), nullable=True)
class ArbitrumSepoliaTransaction(Base): # type: ignore
__tablename__ = "arbitrum_sepolia_transactions"
hash = Column(
VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
)
block_number = Column(
BigInteger,
ForeignKey("arbitrum_sepolia_blocks.block_number", ondelete="CASCADE"),
nullable=False,
index=True,
)
from_address = Column(VARCHAR(256), index=True)
to_address = Column(VARCHAR(256), index=True)
gas = Column(Numeric(precision=78, scale=0), index=True)
gas_price = Column(Numeric(precision=78, scale=0), index=True)
max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
input = Column(Text)
nonce = Column(VARCHAR(256))
transaction_index = Column(BigInteger)
transaction_type = Column(Integer, nullable=True)
value = Column(Numeric(precision=78, scale=0), index=True)
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
y_parity = Column(BigInteger, nullable=True)
class ArbitrumSepoliaLabel(Base): # type: ignore
__tablename__ = "arbitrum_sepolia_labels"
id = Column(
UUID(as_uuid=True),
primary_key=True,
default=uuid.uuid4,
unique=True,
nullable=False,
)
label = Column(VARCHAR(256), nullable=False, index=True)
block_number = Column(
BigInteger,
nullable=True,
index=True,
)
address = Column(
VARCHAR(256),
nullable=True,
index=True,
)
transaction_hash = Column(
VARCHAR(256),
nullable=True,
index=True,
)
label_data = Column(JSONB, nullable=True)
block_timestamp = Column(BigInteger, index=True)
log_index = Column(Integer, nullable=True)
created_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
class ESDFunctionSignature(Base): # type: ignore
"""
Function signature from blockchain (Ethereum/Polygon) Signature Database.

10
moonstreamdb/moonstreamdb/networks.py
Wyświetl plik

@ -27,6 +27,9 @@ from .models import (
ArbitrumNovaBlock,
ArbitrumNovaTransaction,
ArbitrumNovaLabel,
ArbitrumSepoliaBlock,
ArbitrumSepoliaTransaction,
ArbitrumSepoliaLabel,
)
@ -39,6 +42,7 @@ class Network(Enum):
zksync_era_testnet = "zksync_era_testnet"
zksync_era = "zksync_era"
arbitrum_nova = "arbitrum_nova"
arbitrum_sepolia = "arbitrum_sepolia"
tx_raw_types = Union[
@ -50,6 +54,7 @@ tx_raw_types = Union[
ZkSyncEraTestnetTransaction,
ZkSyncEraTransaction,
ArbitrumNovaTransaction,
ArbitrumSepoliaTransaction,
]
MODELS: Dict[Network, Dict[str, Base]] = {
@ -93,4 +98,9 @@ MODELS: Dict[Network, Dict[str, Base]] = {
"labels": ArbitrumNovaLabel,
"transactions": ArbitrumNovaTransaction,
},
Network.arbitrum_sepolia: {
"blocks": ArbitrumSepoliaBlock,
"labels": ArbitrumSepoliaLabel,
"transactions": ArbitrumSepoliaTransaction,
},
}

2
moonstreamdb/moonstreamdb/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream database version.
"""
MOONSTREAMDB_VERSION = "0.3.6"
MOONSTREAMDB_VERSION = "0.3.7"