kopia lustrzana https://github.com/bugout-dev/moonstream
Merge pull request #1009 from moonstream-to/arb-nova-crawlers
Arbitrum nova crawlers and modelpull/1012/head^2 moonstreamdb/v0.3.6
commit
28dc39c71d
|
@ -0,0 +1,11 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Fill missing blocks at Arbitrum Nova database
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||||
|
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||||
|
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_nova -n
|
||||||
|
CPUWeight=50
|
||||||
|
SyslogIdentifier=arbitrum-nova-missing
|
|
@ -0,0 +1,9 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Fill missing blocks at Arbitrum Nova database
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=120s
|
||||||
|
OnUnitActiveSec=15m
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
|
@ -0,0 +1,17 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Arbitrum Nova moonworm crawler
|
||||||
|
After=network.target
|
||||||
|
StartLimitIntervalSec=300
|
||||||
|
StartLimitBurst=3
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||||
|
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=15s
|
||||||
|
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_nova --confirmations 10 --min-blocks-batch 20
|
||||||
|
CPUWeight=70
|
||||||
|
SyslogIdentifier=arbitrum-nova-moonworm-crawler
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
|
@ -0,0 +1,17 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Arbitrum Nova block with transactions synchronizer
|
||||||
|
StartLimitIntervalSec=300
|
||||||
|
StartLimitBurst=3
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=15s
|
||||||
|
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
|
||||||
|
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
|
||||||
|
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_nova -c 10 -j 2
|
||||||
|
CPUWeight=90
|
||||||
|
SyslogIdentifier=arbitrum-nova-synchronize
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
|
@ -122,6 +122,12 @@ ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="zksync-era-testnet-
|
||||||
ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-testnet-historical-crawl-events.service"
|
ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-testnet-historical-crawl-events.service"
|
||||||
ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-testnet-historical-crawl-events.timer"
|
ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-testnet-historical-crawl-events.timer"
|
||||||
|
|
||||||
|
# Arbitrum Nova
|
||||||
|
ARBITRUM_NOVA_MISSING_SERVICE_FILE="arbitrum-nova-missing.service"
|
||||||
|
ARBITRUM_NOVA_MISSING_TIMER_FILE="arbitrum-nova-missing.timer"
|
||||||
|
ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-nova-moonworm-crawler.service"
|
||||||
|
ARBITRUM_NOVA_SYNCHRONIZE_SERVICE="arbitrum-nova-synchronize.service"
|
||||||
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
echo
|
echo
|
||||||
|
@ -584,3 +590,29 @@ cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "/home/ubuntu/.config/sys
|
||||||
cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}"
|
cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}"
|
||||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}"
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}"
|
||||||
|
|
||||||
|
# Arbitrum Nova
|
||||||
|
echo
|
||||||
|
echo
|
||||||
|
echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova block with transactions syncronizer service definition with ${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}"
|
||||||
|
chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}"
|
||||||
|
cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}"
|
||||||
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||||
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo
|
||||||
|
echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova missing service and timer with: ${ARBITRUM_NOVA_MISSING_SERVICE_FILE}, ${ARBITRUM_NOVA_MISSING_TIMER_FILE}"
|
||||||
|
chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_TIMER_FILE}"
|
||||||
|
cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MISSING_SERVICE_FILE}"
|
||||||
|
cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MISSING_TIMER_FILE}"
|
||||||
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||||
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_MISSING_TIMER_FILE}"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo
|
||||||
|
echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova moonworm crawler service definition with ${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||||
|
chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||||
|
cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||||
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||||
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||||
|
|
|
@ -9,7 +9,7 @@ Restart=on-failure
|
||||||
RestartSec=15s
|
RestartSec=15s
|
||||||
WorkingDirectory=/home/ubuntu/
|
WorkingDirectory=/home/ubuntu/
|
||||||
EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env
|
EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env
|
||||||
ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service
|
ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service
|
||||||
CPUWeight=90
|
CPUWeight=90
|
||||||
SyslogIdentifier=monitoring-crawlers
|
SyslogIdentifier=monitoring-crawlers
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
import logging
|
import logging
|
||||||
from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait
|
from concurrent.futures import Future, ThreadPoolExecutor, wait
|
||||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
|
@ -21,6 +21,7 @@ from web3.types import BlockData
|
||||||
from .data import DateRange
|
from .data import DateRange
|
||||||
from .db import yield_db_session, yield_db_session_ctx
|
from .db import yield_db_session, yield_db_session_ctx
|
||||||
from .settings import (
|
from .settings import (
|
||||||
|
MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI,
|
||||||
MOONSTREAM_CRAWL_WORKERS,
|
MOONSTREAM_CRAWL_WORKERS,
|
||||||
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
|
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
|
||||||
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
|
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
|
||||||
|
@ -76,6 +77,8 @@ def connect(
|
||||||
web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
|
web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
|
||||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||||
web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI
|
web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI
|
||||||
|
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
|
||||||
|
web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI
|
||||||
else:
|
else:
|
||||||
raise Exception("Wrong blockchain type provided for web3 URI")
|
raise Exception("Wrong blockchain type provided for web3 URI")
|
||||||
|
|
||||||
|
@ -94,6 +97,15 @@ def connect(
|
||||||
return web3_client
|
return web3_client
|
||||||
|
|
||||||
|
|
||||||
|
def hex_to_int(hex_str: Optional[str] = None) -> Optional[int]:
|
||||||
|
if hex_str is None:
|
||||||
|
return None
|
||||||
|
elif hex_str.startswith("0x"):
|
||||||
|
return int(hex_str, 16)
|
||||||
|
else:
|
||||||
|
return int(hex_str)
|
||||||
|
|
||||||
|
|
||||||
def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) -> None:
|
def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) -> None:
|
||||||
"""
|
"""
|
||||||
Add block if doesn't presented in database.
|
Add block if doesn't presented in database.
|
||||||
|
@ -145,6 +157,12 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
|
||||||
if block.get("l1BatchTimestamp") is not None
|
if block.get("l1BatchTimestamp") is not None
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
|
||||||
|
block_obj.sha3_uncles = block.get("sha3Uncles", "")
|
||||||
|
block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber"))
|
||||||
|
block_obj.send_count = hex_to_int(block.get("sendCount"))
|
||||||
|
block_obj.send_root = block.get("sendRoot", "")
|
||||||
|
block_obj.mix_hash = block.get("mixHash", "")
|
||||||
|
|
||||||
db_session.add(block_obj)
|
db_session.add(block_obj)
|
||||||
|
|
||||||
|
@ -188,6 +206,8 @@ def add_block_transactions(
|
||||||
if tx.get("l1BatchTxIndex") is not None
|
if tx.get("l1BatchTxIndex") is not None
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
|
||||||
|
tx_obj.y_parity = hex_to_int(tx.get("yParity"))
|
||||||
|
|
||||||
db_session.add(tx_obj)
|
db_session.add(tx_obj)
|
||||||
|
|
||||||
|
|
|
@ -6,13 +6,17 @@ from typing import Dict, List, Optional, Tuple
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from moonstreamdb.blockchain import AvailableBlockchainType
|
from moonstreamdb.blockchain import AvailableBlockchainType
|
||||||
|
from moonstreamdb.networks import Network
|
||||||
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
|
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
|
||||||
MoonstreamEthereumStateProvider,
|
MoonstreamEthereumStateProvider,
|
||||||
)
|
)
|
||||||
from moonstreamdb.networks import Network
|
|
||||||
from sqlalchemy.orm.session import Session
|
from sqlalchemy.orm.session import Session
|
||||||
from web3 import Web3
|
from web3 import Web3
|
||||||
|
|
||||||
|
from ..settings import (
|
||||||
|
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
|
||||||
|
HISTORICAL_CRAWLER_STATUSES,
|
||||||
|
)
|
||||||
from .crawler import (
|
from .crawler import (
|
||||||
EventCrawlJob,
|
EventCrawlJob,
|
||||||
FunctionCallCrawlJob,
|
FunctionCallCrawlJob,
|
||||||
|
@ -29,10 +33,6 @@ from .crawler import (
|
||||||
from .db import add_events_to_session, add_function_calls_to_session, commit_session
|
from .db import add_events_to_session, add_function_calls_to_session, commit_session
|
||||||
from .event_crawler import _crawl_events
|
from .event_crawler import _crawl_events
|
||||||
from .function_call_crawler import _crawl_functions
|
from .function_call_crawler import _crawl_functions
|
||||||
from ..settings import (
|
|
||||||
HISTORICAL_CRAWLER_STATUSES,
|
|
||||||
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
|
|
||||||
)
|
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -134,6 +134,8 @@ def continuous_crawler(
|
||||||
network = Network.zksync_era_testnet
|
network = Network.zksync_era_testnet
|
||||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||||
network = Network.zksync_era
|
network = Network.zksync_era
|
||||||
|
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
|
||||||
|
network = Network.arbitrum_nova
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
||||||
|
|
||||||
|
|
|
@ -5,24 +5,24 @@ import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Any, Callable, Dict, List, Optional, cast, Union, Tuple
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from bugout.data import BugoutSearchResult, BugoutJournalEntries
|
from bugout.data import BugoutJournalEntries, BugoutSearchResult
|
||||||
from eth_typing.evm import ChecksumAddress
|
from eth_typing.evm import ChecksumAddress
|
||||||
from moonstreamdb.blockchain import AvailableBlockchainType
|
from moonstreamdb.blockchain import AvailableBlockchainType
|
||||||
from web3.main import Web3
|
|
||||||
from moonworm.deployment import find_deployment_block # type: ignore
|
from moonworm.deployment import find_deployment_block # type: ignore
|
||||||
|
from web3.main import Web3
|
||||||
|
|
||||||
from ..blockchain import connect
|
from ..blockchain import connect
|
||||||
from ..reporter import reporter
|
from ..reporter import reporter
|
||||||
from ..settings import (
|
from ..settings import (
|
||||||
BUGOUT_REQUEST_TIMEOUT_SECONDS,
|
BUGOUT_REQUEST_TIMEOUT_SECONDS,
|
||||||
|
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
|
||||||
|
HISTORICAL_CRAWLER_STATUSES,
|
||||||
MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||||
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
|
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
|
||||||
bugout_client,
|
bugout_client,
|
||||||
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
|
|
||||||
HISTORICAL_CRAWLER_STATUSES,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
@ -37,6 +37,7 @@ class SubscriptionTypes(Enum):
|
||||||
WYRM_BLOCKCHAIN = "wyrm_smartcontract"
|
WYRM_BLOCKCHAIN = "wyrm_smartcontract"
|
||||||
ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
|
ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
|
||||||
ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract"
|
ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract"
|
||||||
|
ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract"
|
||||||
|
|
||||||
|
|
||||||
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
|
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
|
||||||
|
@ -145,6 +146,8 @@ def blockchain_type_to_subscription_type(
|
||||||
return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
|
return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
|
||||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||||
return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN
|
return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN
|
||||||
|
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
|
||||||
|
return SubscriptionTypes.ARBITRUM_NOVA_BLOCKCHAIN
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,7 @@ import logging
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
||||||
from moonstreamdb.blockchain import AvailableBlockchainType
|
from moonstreamdb.blockchain import AvailableBlockchainType
|
||||||
|
from moonstreamdb.networks import Network # type: ignore
|
||||||
from moonworm.crawler.function_call_crawler import ( # type: ignore
|
from moonworm.crawler.function_call_crawler import ( # type: ignore
|
||||||
ContractFunctionCall,
|
ContractFunctionCall,
|
||||||
FunctionCallCrawler,
|
FunctionCallCrawler,
|
||||||
|
@ -9,7 +10,6 @@ from moonworm.crawler.function_call_crawler import ( # type: ignore
|
||||||
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
|
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
|
||||||
MoonstreamEthereumStateProvider,
|
MoonstreamEthereumStateProvider,
|
||||||
)
|
)
|
||||||
from moonstreamdb.networks import Network # type: ignore
|
|
||||||
from moonworm.watch import MockState # type: ignore
|
from moonworm.watch import MockState # type: ignore
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from web3 import Web3
|
from web3 import Web3
|
||||||
|
@ -72,6 +72,8 @@ def function_call_crawler(
|
||||||
network = Network.zksync_era_testnet
|
network = Network.zksync_era_testnet
|
||||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||||
network = Network.zksync_era
|
network = Network.zksync_era
|
||||||
|
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
|
||||||
|
network = Network.arbitrum_nova
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
raise ValueError(f"Unknown blockchain type: {blockchain_type}")
|
||||||
|
|
||||||
|
|
|
@ -5,10 +5,10 @@ from uuid import UUID
|
||||||
|
|
||||||
from eth_typing.evm import ChecksumAddress
|
from eth_typing.evm import ChecksumAddress
|
||||||
from moonstreamdb.blockchain import AvailableBlockchainType
|
from moonstreamdb.blockchain import AvailableBlockchainType
|
||||||
|
from moonstreamdb.networks import Network # type: ignore
|
||||||
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
|
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
|
||||||
MoonstreamEthereumStateProvider,
|
MoonstreamEthereumStateProvider,
|
||||||
)
|
)
|
||||||
from moonstreamdb.networks import Network # type: ignore
|
|
||||||
from sqlalchemy.orm.session import Session
|
from sqlalchemy.orm.session import Session
|
||||||
from web3 import Web3
|
from web3 import Web3
|
||||||
|
|
||||||
|
@ -19,7 +19,7 @@ from .crawler import (
|
||||||
update_entries_status_and_progress,
|
update_entries_status_and_progress,
|
||||||
)
|
)
|
||||||
from .db import add_events_to_session, add_function_calls_to_session, commit_session
|
from .db import add_events_to_session, add_function_calls_to_session, commit_session
|
||||||
from .event_crawler import _crawl_events, _autoscale_crawl_events
|
from .event_crawler import _autoscale_crawl_events, _crawl_events
|
||||||
from .function_call_crawler import _crawl_functions
|
from .function_call_crawler import _crawl_functions
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
@ -65,6 +65,8 @@ def historical_crawler(
|
||||||
network = Network.zksync_era_testnet
|
network = Network.zksync_era_testnet
|
||||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||||
network = Network.zksync_era
|
network = Network.zksync_era
|
||||||
|
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
|
||||||
|
network = Network.arbitrum_nova
|
||||||
else:
|
else:
|
||||||
raise Exception("Unsupported blockchain type provided")
|
raise Exception("Unsupported blockchain type provided")
|
||||||
|
|
||||||
|
|
|
@ -122,6 +122,14 @@ MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI = os.environ.get(
|
||||||
if MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI == "":
|
if MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI == "":
|
||||||
raise Exception("MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI env variable is not set")
|
raise Exception("MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI env variable is not set")
|
||||||
|
|
||||||
|
MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI = os.environ.get(
|
||||||
|
"MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI", ""
|
||||||
|
)
|
||||||
|
if MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI == "":
|
||||||
|
raise Exception(
|
||||||
|
"MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI env variable is not set"
|
||||||
|
)
|
||||||
|
|
||||||
MOONSTREAM_CRAWL_WORKERS = 4
|
MOONSTREAM_CRAWL_WORKERS = 4
|
||||||
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
|
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -8,7 +8,7 @@ import logging
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Any, Callable, cast, Dict, List, Optional, Union
|
from typing import Any, Callable, Dict, List, Optional, Union, cast
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
import boto3 # type: ignore
|
import boto3 # type: ignore
|
||||||
|
@ -23,8 +23,9 @@ from moonstreamdb.blockchain import (
|
||||||
get_label_model,
|
get_label_model,
|
||||||
get_transaction_model,
|
get_transaction_model,
|
||||||
)
|
)
|
||||||
from sqlalchemy import and_, distinct, extract, func, text
|
from sqlalchemy import and_
|
||||||
from sqlalchemy import cast as sqlalchemy_cast
|
from sqlalchemy import cast as sqlalchemy_cast
|
||||||
|
from sqlalchemy import distinct, extract, func, text
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from sqlalchemy.sql.operators import in_op
|
from sqlalchemy.sql.operators import in_op
|
||||||
from web3 import Web3
|
from web3 import Web3
|
||||||
|
@ -54,6 +55,7 @@ subscription_id_by_blockchain = {
|
||||||
"wyrm": "wyrm_smartcontract",
|
"wyrm": "wyrm_smartcontract",
|
||||||
"zksync_era_testnet": "zksync_era_testnet_smartcontract",
|
"zksync_era_testnet": "zksync_era_testnet_smartcontract",
|
||||||
"zksync_era": "zksync_era_smartcontract",
|
"zksync_era": "zksync_era_smartcontract",
|
||||||
|
"arbitrum_nova": "arbitrum_nova_smartcontract",
|
||||||
}
|
}
|
||||||
|
|
||||||
blockchain_by_subscription_id = {
|
blockchain_by_subscription_id = {
|
||||||
|
@ -63,6 +65,7 @@ blockchain_by_subscription_id = {
|
||||||
"xdai_blockchain": "xdai",
|
"xdai_blockchain": "xdai",
|
||||||
"wyrm_blockchain": "wyrm",
|
"wyrm_blockchain": "wyrm",
|
||||||
"zksync_era_testnet_blockchain": "zksync_era_testnet",
|
"zksync_era_testnet_blockchain": "zksync_era_testnet",
|
||||||
|
"arbitrum_nova_blockchain": "arbitrum_nova",
|
||||||
"ethereum_smartcontract": "ethereum",
|
"ethereum_smartcontract": "ethereum",
|
||||||
"polygon_smartcontract": "polygon",
|
"polygon_smartcontract": "polygon",
|
||||||
"mumbai_smartcontract": "mumbai",
|
"mumbai_smartcontract": "mumbai",
|
||||||
|
@ -70,6 +73,7 @@ blockchain_by_subscription_id = {
|
||||||
"wyrm_smartcontract": "wyrm",
|
"wyrm_smartcontract": "wyrm",
|
||||||
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
|
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
|
||||||
"zksync_era_smartcontract": "zksync_era",
|
"zksync_era_smartcontract": "zksync_era",
|
||||||
|
"arbitrum_nova_smartcontract": "arbitrum_nova",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,4 +2,4 @@
|
||||||
Moonstream crawlers version.
|
Moonstream crawlers version.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
MOONCRAWL_VERSION = "0.3.5"
|
MOONCRAWL_VERSION = "0.3.6"
|
||||||
|
|
|
@ -27,6 +27,8 @@ export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_mumb
|
||||||
export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
|
export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
|
||||||
export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_node>"
|
export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_node>"
|
||||||
export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_testnet_node>"
|
export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_testnet_node>"
|
||||||
|
export MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_node>"
|
||||||
|
export MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_arbitrum_nova_node>"
|
||||||
export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
|
export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
|
||||||
|
|
||||||
# AWS environment variables
|
# AWS environment variables
|
||||||
|
|
|
@ -37,7 +37,7 @@ setup(
|
||||||
"bugout>=0.2.13",
|
"bugout>=0.2.13",
|
||||||
"chardet",
|
"chardet",
|
||||||
"fastapi",
|
"fastapi",
|
||||||
"moonstreamdb>=0.3.5",
|
"moonstreamdb>=0.3.6",
|
||||||
"moonstream>=0.1.1",
|
"moonstream>=0.1.1",
|
||||||
"moonworm[moonstream]>=0.6.2",
|
"moonworm[moonstream]>=0.6.2",
|
||||||
"humbug",
|
"humbug",
|
||||||
|
|
|
@ -46,6 +46,9 @@ from moonstreamdb.models import (
|
||||||
ZkSyncEraTestnetBlock,
|
ZkSyncEraTestnetBlock,
|
||||||
ZkSyncEraTestnetLabel,
|
ZkSyncEraTestnetLabel,
|
||||||
ZkSyncEraTestnetTransaction,
|
ZkSyncEraTestnetTransaction,
|
||||||
|
ArbitrumNovaBlock,
|
||||||
|
ArbitrumNovaTransaction,
|
||||||
|
ArbitrumNovaLabel,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -72,6 +75,9 @@ def include_symbol(tablename, schema):
|
||||||
ZkSyncEraTestnetBlock.__tablename__,
|
ZkSyncEraTestnetBlock.__tablename__,
|
||||||
ZkSyncEraTestnetLabel.__tablename__,
|
ZkSyncEraTestnetLabel.__tablename__,
|
||||||
ZkSyncEraTestnetTransaction.__tablename__,
|
ZkSyncEraTestnetTransaction.__tablename__,
|
||||||
|
ArbitrumNovaBlock.__tablename__,
|
||||||
|
ArbitrumNovaTransaction.__tablename__,
|
||||||
|
ArbitrumNovaLabel.__tablename__,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,118 @@
|
||||||
|
"""Arbitrum Nova blockchain support
|
||||||
|
|
||||||
|
Revision ID: 2391c3cc5050
|
||||||
|
Revises: 0f8ee1ebb45f
|
||||||
|
Create Date: 2024-01-30 11:55:24.251107
|
||||||
|
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = '2391c3cc5050'
|
||||||
|
down_revision = '0f8ee1ebb45f'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.create_table('arbitrum_nova_blocks',
|
||||||
|
sa.Column('block_number', sa.BigInteger(), nullable=False),
|
||||||
|
sa.Column('difficulty', sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True),
|
||||||
|
sa.Column('gas_limit', sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column('gas_used', sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||||
|
sa.Column('hash', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True),
|
||||||
|
sa.Column('miner', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('uncles', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('size', sa.Integer(), nullable=True),
|
||||||
|
sa.Column('state_root', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('timestamp', sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
|
||||||
|
sa.Column('sha3_uncles', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('l1_block_number', sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column('send_count', sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column('send_root', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('mix_hash', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.PrimaryKeyConstraint('block_number', name=op.f('pk_arbitrum_nova_blocks'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_blocks_block_number'), 'arbitrum_nova_blocks', ['block_number'], unique=True)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_blocks_hash'), 'arbitrum_nova_blocks', ['hash'], unique=False)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_blocks_timestamp'), 'arbitrum_nova_blocks', ['timestamp'], unique=False)
|
||||||
|
op.create_table('arbitrum_nova_labels',
|
||||||
|
sa.Column('id', sa.UUID(), nullable=False),
|
||||||
|
sa.Column('label', sa.VARCHAR(length=256), nullable=False),
|
||||||
|
sa.Column('block_number', sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column('address', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
||||||
|
sa.Column('block_timestamp', sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column('log_index', sa.Integer(), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_arbitrum_nova_labels')),
|
||||||
|
sa.UniqueConstraint('id', name=op.f('uq_arbitrum_nova_labels_id'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_labels_address'), 'arbitrum_nova_labels', ['address'], unique=False)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_labels_block_number'), 'arbitrum_nova_labels', ['block_number'], unique=False)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_labels_block_timestamp'), 'arbitrum_nova_labels', ['block_timestamp'], unique=False)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_labels_label'), 'arbitrum_nova_labels', ['label'], unique=False)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_labels_transaction_hash'), 'arbitrum_nova_labels', ['transaction_hash'], unique=False)
|
||||||
|
op.create_table('arbitrum_nova_transactions',
|
||||||
|
sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
|
||||||
|
sa.Column('block_number', sa.BigInteger(), nullable=False),
|
||||||
|
sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||||
|
sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||||
|
sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||||
|
sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||||
|
sa.Column('input', sa.Text(), nullable=True),
|
||||||
|
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
|
||||||
|
sa.Column('transaction_index', sa.BigInteger(), nullable=True),
|
||||||
|
sa.Column('transaction_type', sa.Integer(), nullable=True),
|
||||||
|
sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
|
||||||
|
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
|
||||||
|
sa.Column('y_parity', sa.BigInteger(), nullable=True),
|
||||||
|
sa.ForeignKeyConstraint(['block_number'], ['arbitrum_nova_blocks.block_number'], name=op.f('fk_arbitrum_nova_transactions_block_number_arbitrum_nova_blocks'), ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('hash', name=op.f('pk_arbitrum_nova_transactions'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_transactions_block_number'), 'arbitrum_nova_transactions', ['block_number'], unique=False)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_transactions_from_address'), 'arbitrum_nova_transactions', ['from_address'], unique=False)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_transactions_gas'), 'arbitrum_nova_transactions', ['gas'], unique=False)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_transactions_gas_price'), 'arbitrum_nova_transactions', ['gas_price'], unique=False)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_transactions_hash'), 'arbitrum_nova_transactions', ['hash'], unique=True)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_transactions_to_address'), 'arbitrum_nova_transactions', ['to_address'], unique=False)
|
||||||
|
op.create_index(op.f('ix_arbitrum_nova_transactions_value'), 'arbitrum_nova_transactions', ['value'], unique=False)
|
||||||
|
# ### end Alembic commands ###
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_transactions_value'), table_name='arbitrum_nova_transactions')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_transactions_to_address'), table_name='arbitrum_nova_transactions')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_transactions_hash'), table_name='arbitrum_nova_transactions')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_transactions_gas_price'), table_name='arbitrum_nova_transactions')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_transactions_gas'), table_name='arbitrum_nova_transactions')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_transactions_from_address'), table_name='arbitrum_nova_transactions')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_transactions_block_number'), table_name='arbitrum_nova_transactions')
|
||||||
|
op.drop_table('arbitrum_nova_transactions')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_labels_transaction_hash'), table_name='arbitrum_nova_labels')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_labels_label'), table_name='arbitrum_nova_labels')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_labels_block_timestamp'), table_name='arbitrum_nova_labels')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_labels_block_number'), table_name='arbitrum_nova_labels')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_labels_address'), table_name='arbitrum_nova_labels')
|
||||||
|
op.drop_table('arbitrum_nova_labels')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_blocks_timestamp'), table_name='arbitrum_nova_blocks')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_blocks_hash'), table_name='arbitrum_nova_blocks')
|
||||||
|
op.drop_index(op.f('ix_arbitrum_nova_blocks_block_number'), table_name='arbitrum_nova_blocks')
|
||||||
|
op.drop_table('arbitrum_nova_blocks')
|
||||||
|
# ### end Alembic commands ###
|
|
@ -23,6 +23,9 @@ from .models import (
|
||||||
ZkSyncEraBlock,
|
ZkSyncEraBlock,
|
||||||
ZkSyncEraLabel,
|
ZkSyncEraLabel,
|
||||||
ZkSyncEraTransaction,
|
ZkSyncEraTransaction,
|
||||||
|
ArbitrumNovaBlock,
|
||||||
|
ArbitrumNovaTransaction,
|
||||||
|
ArbitrumNovaLabel,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -34,6 +37,7 @@ class AvailableBlockchainType(Enum):
|
||||||
WYRM = "wyrm"
|
WYRM = "wyrm"
|
||||||
ZKSYNC_ERA_TESTNET = "zksync_era_testnet"
|
ZKSYNC_ERA_TESTNET = "zksync_era_testnet"
|
||||||
ZKSYNC_ERA = "zksync_era"
|
ZKSYNC_ERA = "zksync_era"
|
||||||
|
ARBITRUM_NOVA = "arbitrum_nova"
|
||||||
|
|
||||||
|
|
||||||
def get_block_model(
|
def get_block_model(
|
||||||
|
@ -47,10 +51,11 @@ def get_block_model(
|
||||||
WyrmBlock,
|
WyrmBlock,
|
||||||
ZkSyncEraTestnetBlock,
|
ZkSyncEraTestnetBlock,
|
||||||
ZkSyncEraBlock,
|
ZkSyncEraBlock,
|
||||||
|
ArbitrumNovaBlock,
|
||||||
]
|
]
|
||||||
]:
|
]:
|
||||||
"""
|
"""
|
||||||
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet
|
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock
|
||||||
set proper blocks model.
|
set proper blocks model.
|
||||||
"""
|
"""
|
||||||
block_model: Type[
|
block_model: Type[
|
||||||
|
@ -62,6 +67,7 @@ def get_block_model(
|
||||||
WyrmBlock,
|
WyrmBlock,
|
||||||
ZkSyncEraTestnetBlock,
|
ZkSyncEraTestnetBlock,
|
||||||
ZkSyncEraBlock,
|
ZkSyncEraBlock,
|
||||||
|
ArbitrumNovaBlock,
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
if blockchain_type == AvailableBlockchainType.ETHEREUM:
|
if blockchain_type == AvailableBlockchainType.ETHEREUM:
|
||||||
|
@ -78,6 +84,8 @@ def get_block_model(
|
||||||
block_model = ZkSyncEraTestnetBlock
|
block_model = ZkSyncEraTestnetBlock
|
||||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||||
block_model = ZkSyncEraBlock
|
block_model = ZkSyncEraBlock
|
||||||
|
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
|
||||||
|
block_model = ArbitrumNovaBlock
|
||||||
else:
|
else:
|
||||||
raise Exception("Unsupported blockchain type provided")
|
raise Exception("Unsupported blockchain type provided")
|
||||||
|
|
||||||
|
@ -95,10 +103,11 @@ def get_label_model(
|
||||||
WyrmLabel,
|
WyrmLabel,
|
||||||
ZkSyncEraTestnetLabel,
|
ZkSyncEraTestnetLabel,
|
||||||
ZkSyncEraLabel,
|
ZkSyncEraLabel,
|
||||||
|
ArbitrumNovaLabel,
|
||||||
]
|
]
|
||||||
]:
|
]:
|
||||||
"""
|
"""
|
||||||
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet
|
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel
|
||||||
set proper block label model.
|
set proper block label model.
|
||||||
"""
|
"""
|
||||||
label_model: Type[
|
label_model: Type[
|
||||||
|
@ -110,6 +119,7 @@ def get_label_model(
|
||||||
WyrmLabel,
|
WyrmLabel,
|
||||||
ZkSyncEraTestnetLabel,
|
ZkSyncEraTestnetLabel,
|
||||||
ZkSyncEraLabel,
|
ZkSyncEraLabel,
|
||||||
|
ArbitrumNovaLabel,
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
if blockchain_type == AvailableBlockchainType.ETHEREUM:
|
if blockchain_type == AvailableBlockchainType.ETHEREUM:
|
||||||
|
@ -126,6 +136,8 @@ def get_label_model(
|
||||||
label_model = ZkSyncEraTestnetLabel
|
label_model = ZkSyncEraTestnetLabel
|
||||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||||
label_model = ZkSyncEraLabel
|
label_model = ZkSyncEraLabel
|
||||||
|
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
|
||||||
|
label_model = ArbitrumNovaLabel
|
||||||
else:
|
else:
|
||||||
raise Exception("Unsupported blockchain type provided")
|
raise Exception("Unsupported blockchain type provided")
|
||||||
|
|
||||||
|
@ -143,10 +155,11 @@ def get_transaction_model(
|
||||||
WyrmTransaction,
|
WyrmTransaction,
|
||||||
ZkSyncEraTestnetTransaction,
|
ZkSyncEraTestnetTransaction,
|
||||||
ZkSyncEraTransaction,
|
ZkSyncEraTransaction,
|
||||||
|
ArbitrumNovaTransaction,
|
||||||
]
|
]
|
||||||
]:
|
]:
|
||||||
"""
|
"""
|
||||||
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet
|
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction
|
||||||
set proper block transactions model.
|
set proper block transactions model.
|
||||||
"""
|
"""
|
||||||
transaction_model: Type[
|
transaction_model: Type[
|
||||||
|
@ -158,6 +171,7 @@ def get_transaction_model(
|
||||||
WyrmTransaction,
|
WyrmTransaction,
|
||||||
ZkSyncEraTestnetTransaction,
|
ZkSyncEraTestnetTransaction,
|
||||||
ZkSyncEraTransaction,
|
ZkSyncEraTransaction,
|
||||||
|
ArbitrumNovaTransaction,
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
if blockchain_type == AvailableBlockchainType.ETHEREUM:
|
if blockchain_type == AvailableBlockchainType.ETHEREUM:
|
||||||
|
@ -174,6 +188,8 @@ def get_transaction_model(
|
||||||
transaction_model = ZkSyncEraTestnetTransaction
|
transaction_model = ZkSyncEraTestnetTransaction
|
||||||
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
|
||||||
transaction_model = ZkSyncEraTransaction
|
transaction_model = ZkSyncEraTransaction
|
||||||
|
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
|
||||||
|
transaction_model = ArbitrumNovaTransaction
|
||||||
else:
|
else:
|
||||||
raise Exception("Unsupported blockchain type provided")
|
raise Exception("Unsupported blockchain type provided")
|
||||||
|
|
||||||
|
|
|
@ -214,21 +214,6 @@ class PolygonTransaction(Base): # type: ignore
|
||||||
|
|
||||||
|
|
||||||
class PolygonLabel(Base): # type: ignore
|
class PolygonLabel(Base): # type: ignore
|
||||||
"""
|
|
||||||
Example of label_data:
|
|
||||||
{
|
|
||||||
"label": "ERC20",
|
|
||||||
"label_data": {
|
|
||||||
"name": "Uniswap",
|
|
||||||
"symbol": "UNI"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Exchange"
|
|
||||||
"label_data": {...}
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
__tablename__ = "polygon_labels"
|
__tablename__ = "polygon_labels"
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
@ -445,21 +430,6 @@ class XDaiTransaction(Base): # type: ignore
|
||||||
|
|
||||||
|
|
||||||
class XDaiLabel(Base): # type: ignore
|
class XDaiLabel(Base): # type: ignore
|
||||||
"""
|
|
||||||
Example of label_data:
|
|
||||||
{
|
|
||||||
"label": "ERC20",
|
|
||||||
"label_data": {
|
|
||||||
"name": "Uniswap",
|
|
||||||
"symbol": "UNI"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Exchange"
|
|
||||||
"label_data": {...}
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
__tablename__ = "xdai_labels"
|
__tablename__ = "xdai_labels"
|
||||||
|
|
||||||
id = Column(
|
id = Column(
|
||||||
|
@ -551,21 +521,6 @@ class WyrmTransaction(Base): # type: ignore
|
||||||
|
|
||||||
|
|
||||||
class WyrmLabel(Base): # type: ignore
|
class WyrmLabel(Base): # type: ignore
|
||||||
"""
|
|
||||||
Example of label_data:
|
|
||||||
{
|
|
||||||
"label": "ERC20",
|
|
||||||
"label_data": {
|
|
||||||
"name": "Uniswap",
|
|
||||||
"symbol": "UNI"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Exchange"
|
|
||||||
"label_data": {...}
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
__tablename__ = "wyrm_labels"
|
__tablename__ = "wyrm_labels"
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
@ -682,21 +637,6 @@ class ZkSyncEraTransaction(Base): # type: ignore
|
||||||
|
|
||||||
|
|
||||||
class ZkSyncEraLabel(Base): # type: ignore
|
class ZkSyncEraLabel(Base): # type: ignore
|
||||||
"""
|
|
||||||
Example of label_data:
|
|
||||||
{
|
|
||||||
"label": "ERC20",
|
|
||||||
"label_data": {
|
|
||||||
"name": "Uniswap",
|
|
||||||
"symbol": "UNI"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Exchange"
|
|
||||||
"label_data": {...}
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
__tablename__ = "zksync_era_labels"
|
__tablename__ = "zksync_era_labels"
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
@ -813,21 +753,6 @@ class ZkSyncEraTestnetTransaction(Base): # type: ignore
|
||||||
|
|
||||||
|
|
||||||
class ZkSyncEraTestnetLabel(Base): # type: ignore
|
class ZkSyncEraTestnetLabel(Base): # type: ignore
|
||||||
"""
|
|
||||||
Example of label_data:
|
|
||||||
{
|
|
||||||
"label": "ERC20",
|
|
||||||
"label_data": {
|
|
||||||
"name": "Uniswap",
|
|
||||||
"symbol": "UNI"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Exchange"
|
|
||||||
"label_data": {...}
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
__tablename__ = "zksync_era_testnet_labels"
|
__tablename__ = "zksync_era_testnet_labels"
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
@ -876,6 +801,106 @@ class ZkSyncEraTestnetLabel(Base): # type: ignore
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ArbitrumNovaBlock(Base): # type: ignore
|
||||||
|
__tablename__ = "arbitrum_nova_blocks"
|
||||||
|
|
||||||
|
block_number = Column(
|
||||||
|
BigInteger, primary_key=True, unique=True, nullable=False, index=True
|
||||||
|
)
|
||||||
|
difficulty = Column(BigInteger)
|
||||||
|
extra_data = Column(VARCHAR(128))
|
||||||
|
gas_limit = Column(BigInteger)
|
||||||
|
gas_used = Column(BigInteger)
|
||||||
|
base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
|
||||||
|
hash = Column(VARCHAR(256), index=True)
|
||||||
|
logs_bloom = Column(VARCHAR(1024))
|
||||||
|
miner = Column(VARCHAR(256))
|
||||||
|
nonce = Column(VARCHAR(256))
|
||||||
|
parent_hash = Column(VARCHAR(256))
|
||||||
|
receipt_root = Column(VARCHAR(256))
|
||||||
|
uncles = Column(VARCHAR(256))
|
||||||
|
size = Column(Integer)
|
||||||
|
state_root = Column(VARCHAR(256))
|
||||||
|
timestamp = Column(BigInteger, index=True)
|
||||||
|
total_difficulty = Column(VARCHAR(256))
|
||||||
|
transactions_root = Column(VARCHAR(256))
|
||||||
|
|
||||||
|
indexed_at = Column(
|
||||||
|
DateTime(timezone=True), server_default=utcnow(), nullable=False
|
||||||
|
)
|
||||||
|
|
||||||
|
sha3_uncles = Column(VARCHAR(256), nullable=True)
|
||||||
|
l1_block_number = Column(BigInteger, nullable=True)
|
||||||
|
send_count = Column(BigInteger, nullable=True)
|
||||||
|
send_root = Column(VARCHAR(256), nullable=True)
|
||||||
|
mix_hash = Column(VARCHAR(256), nullable=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ArbitrumNovaTransaction(Base): # type: ignore
|
||||||
|
__tablename__ = "arbitrum_nova_transactions"
|
||||||
|
|
||||||
|
hash = Column(
|
||||||
|
VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
|
||||||
|
)
|
||||||
|
block_number = Column(
|
||||||
|
BigInteger,
|
||||||
|
ForeignKey("arbitrum_nova_blocks.block_number", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
from_address = Column(VARCHAR(256), index=True)
|
||||||
|
to_address = Column(VARCHAR(256), index=True)
|
||||||
|
gas = Column(Numeric(precision=78, scale=0), index=True)
|
||||||
|
gas_price = Column(Numeric(precision=78, scale=0), index=True)
|
||||||
|
max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
|
||||||
|
max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
|
||||||
|
input = Column(Text)
|
||||||
|
nonce = Column(VARCHAR(256))
|
||||||
|
transaction_index = Column(BigInteger)
|
||||||
|
transaction_type = Column(Integer, nullable=True)
|
||||||
|
value = Column(Numeric(precision=78, scale=0), index=True)
|
||||||
|
|
||||||
|
indexed_at = Column(
|
||||||
|
DateTime(timezone=True), server_default=utcnow(), nullable=False
|
||||||
|
)
|
||||||
|
|
||||||
|
y_parity = Column(BigInteger, nullable=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ArbitrumNovaLabel(Base): # type: ignore
|
||||||
|
__tablename__ = "arbitrum_nova_labels"
|
||||||
|
|
||||||
|
id = Column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
primary_key=True,
|
||||||
|
default=uuid.uuid4,
|
||||||
|
unique=True,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
label = Column(VARCHAR(256), nullable=False, index=True)
|
||||||
|
block_number = Column(
|
||||||
|
BigInteger,
|
||||||
|
nullable=True,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
address = Column(
|
||||||
|
VARCHAR(256),
|
||||||
|
nullable=True,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
transaction_hash = Column(
|
||||||
|
VARCHAR(256),
|
||||||
|
nullable=True,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
label_data = Column(JSONB, nullable=True)
|
||||||
|
block_timestamp = Column(BigInteger, index=True)
|
||||||
|
log_index = Column(Integer, nullable=True)
|
||||||
|
created_at = Column(
|
||||||
|
DateTime(timezone=True), server_default=utcnow(), nullable=False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ESDFunctionSignature(Base): # type: ignore
|
class ESDFunctionSignature(Base): # type: ignore
|
||||||
"""
|
"""
|
||||||
Function signature from blockchain (Ethereum/Polygon) Signature Database.
|
Function signature from blockchain (Ethereum/Polygon) Signature Database.
|
||||||
|
|
|
@ -24,6 +24,9 @@ from .models import (
|
||||||
ZkSyncEraBlock,
|
ZkSyncEraBlock,
|
||||||
ZkSyncEraLabel,
|
ZkSyncEraLabel,
|
||||||
ZkSyncEraTransaction,
|
ZkSyncEraTransaction,
|
||||||
|
ArbitrumNovaBlock,
|
||||||
|
ArbitrumNovaTransaction,
|
||||||
|
ArbitrumNovaLabel,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -35,6 +38,7 @@ class Network(Enum):
|
||||||
wyrm = "wyrm"
|
wyrm = "wyrm"
|
||||||
zksync_era_testnet = "zksync_era_testnet"
|
zksync_era_testnet = "zksync_era_testnet"
|
||||||
zksync_era = "zksync_era"
|
zksync_era = "zksync_era"
|
||||||
|
arbitrum_nova = "arbitrum_nova"
|
||||||
|
|
||||||
|
|
||||||
tx_raw_types = Union[
|
tx_raw_types = Union[
|
||||||
|
@ -45,6 +49,7 @@ tx_raw_types = Union[
|
||||||
XDaiTransaction,
|
XDaiTransaction,
|
||||||
ZkSyncEraTestnetTransaction,
|
ZkSyncEraTestnetTransaction,
|
||||||
ZkSyncEraTransaction,
|
ZkSyncEraTransaction,
|
||||||
|
ArbitrumNovaTransaction,
|
||||||
]
|
]
|
||||||
|
|
||||||
MODELS: Dict[Network, Dict[str, Base]] = {
|
MODELS: Dict[Network, Dict[str, Base]] = {
|
||||||
|
@ -83,4 +88,9 @@ MODELS: Dict[Network, Dict[str, Base]] = {
|
||||||
"labels": ZkSyncEraLabel,
|
"labels": ZkSyncEraLabel,
|
||||||
"transactions": ZkSyncEraTransaction,
|
"transactions": ZkSyncEraTransaction,
|
||||||
},
|
},
|
||||||
|
Network.arbitrum_nova: {
|
||||||
|
"blocks": ArbitrumNovaBlock,
|
||||||
|
"labels": ArbitrumNovaLabel,
|
||||||
|
"transactions": ArbitrumNovaTransaction,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,4 +2,4 @@
|
||||||
Moonstream database version.
|
Moonstream database version.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
MOONSTREAMDB_VERSION = "0.3.5"
|
MOONSTREAMDB_VERSION = "0.3.6"
|
||||||
|
|
Ładowanie…
Reference in New Issue