mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Basic crawlers for arbitrum nova

pull/1009/head
kompotkot 2024-01-31 10:09:04 +00:00
rodzic e1c6f7c13d
commit 683bc121a7
10 zmienionych plików z 61 dodań i 18 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

22
crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -1,5 +1,5 @@
import logging
from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, wait
from concurrent.futures import Future, ThreadPoolExecutor, wait
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
from uuid import UUID
@ -21,6 +21,7 @@ from web3.types import BlockData
from .data import DateRange
from .db import yield_db_session, yield_db_session_ctx
from .settings import (
MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI,
MOONSTREAM_CRAWL_WORKERS,
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
@ -76,6 +77,8 @@ def connect(
web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI
else:
raise Exception("Wrong blockchain type provided for web3 URI")
@ -94,6 +97,15 @@ def connect(
return web3_client
def hex_to_int(hex_str: Optional[str] = None) -> Optional[int]:
if hex_str is None:
return None
elif hex_str.startswith("0x"):
return int(hex_str, 16)
else:
return int(hex_str)
def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) -> None:
"""
Add block if doesn't presented in database.
@ -145,6 +157,12 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
if block.get("l1BatchTimestamp") is not None
else None
)
if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
block_obj.sha3_uncles = block.get("sha3Uncles", "")
block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber"))
block_obj.send_count = hex_to_int(block.get("sendCount"))
block_obj.send_root = block.get("sendRoot", "")
block_obj.mix_hash = block.get("mixHash", "")
db_session.add(block_obj)
@ -188,6 +206,8 @@ def add_block_transactions(
if tx.get("l1BatchTxIndex") is not None
else None
)
if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
tx_obj.y_parity = hex_to_int(tx.get("yParity"))
db_session.add(tx_obj)

12
crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py
Wyświetl plik

@ -6,13 +6,17 @@ from typing import Dict, List, Optional, Tuple
from uuid import UUID
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.networks import Network
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
MoonstreamEthereumStateProvider,
)
from moonstreamdb.networks import Network
from sqlalchemy.orm.session import Session
from web3 import Web3
from ..settings import (
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
HISTORICAL_CRAWLER_STATUSES,
)
from .crawler import (
EventCrawlJob,
FunctionCallCrawlJob,
@ -29,10 +33,6 @@ from .crawler import (
from .db import add_events_to_session, add_function_calls_to_session, commit_session
from .event_crawler import _crawl_events
from .function_call_crawler import _crawl_functions
from ..settings import (
HISTORICAL_CRAWLER_STATUSES,
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -134,6 +134,8 @@ def continuous_crawler(
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

13
crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py
Wyświetl plik

@ -5,24 +5,24 @@ import time
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Any, Callable, Dict, List, Optional, cast, Union, Tuple
from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
from uuid import UUID
from bugout.data import BugoutSearchResult, BugoutJournalEntries
from bugout.data import BugoutJournalEntries, BugoutSearchResult
from eth_typing.evm import ChecksumAddress
from moonstreamdb.blockchain import AvailableBlockchainType
from web3.main import Web3
from moonworm.deployment import find_deployment_block # type: ignore
from web3.main import Web3
from ..blockchain import connect
from ..reporter import reporter
from ..settings import (
BUGOUT_REQUEST_TIMEOUT_SECONDS,
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
HISTORICAL_CRAWLER_STATUSES,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
bugout_client,
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
HISTORICAL_CRAWLER_STATUSES,
)
logging.basicConfig(level=logging.INFO)
@ -37,6 +37,7 @@ class SubscriptionTypes(Enum):
WYRM_BLOCKCHAIN = "wyrm_smartcontract"
ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract"
ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract"
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
@ -145,6 +146,8 @@ def blockchain_type_to_subscription_type(
return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
return SubscriptionTypes.ARBITRUM_NOVA_BLOCKCHAIN
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

4
crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py
Wyświetl plik

@ -2,6 +2,7 @@ import logging
from typing import List
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.networks import Network # type: ignore
from moonworm.crawler.function_call_crawler import ( # type: ignore
ContractFunctionCall,
FunctionCallCrawler,
@ -9,7 +10,6 @@ from moonworm.crawler.function_call_crawler import ( # type: ignore
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
MoonstreamEthereumStateProvider,
)
from moonstreamdb.networks import Network # type: ignore
from moonworm.watch import MockState # type: ignore
from sqlalchemy.orm import Session
from web3 import Web3
@ -72,6 +72,8 @@ def function_call_crawler(
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

6
crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py
Wyświetl plik

@ -5,10 +5,10 @@ from uuid import UUID
from eth_typing.evm import ChecksumAddress
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstreamdb.networks import Network # type: ignore
from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore
MoonstreamEthereumStateProvider,
)
from moonstreamdb.networks import Network # type: ignore
from sqlalchemy.orm.session import Session
from web3 import Web3
@ -19,7 +19,7 @@ from .crawler import (
update_entries_status_and_progress,
)
from .db import add_events_to_session, add_function_calls_to_session, commit_session
from .event_crawler import _crawl_events, _autoscale_crawl_events
from .event_crawler import _autoscale_crawl_events, _crawl_events
from .function_call_crawler import _crawl_functions
logging.basicConfig(level=logging.INFO)
@ -65,6 +65,8 @@ def historical_crawler(
network = Network.zksync_era_testnet
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
network = Network.zksync_era
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
network = Network.arbitrum_nova
else:
raise Exception("Unsupported blockchain type provided")

8
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -122,6 +122,14 @@ MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI = os.environ.get(
if MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI == "":
raise Exception(
"MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI env variable is not set"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
try:

8
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py
Wyświetl plik

@ -8,7 +8,7 @@ import logging
import time
from datetime import datetime, timedelta
from enum import Enum
from typing import Any, Callable, cast, Dict, List, Optional, Union
from typing import Any, Callable, Dict, List, Optional, Union, cast
from uuid import UUID
import boto3 # type: ignore
@ -23,8 +23,9 @@ from moonstreamdb.blockchain import (
get_label_model,
get_transaction_model,
)
from sqlalchemy import and_, distinct, extract, func, text
from sqlalchemy import and_
from sqlalchemy import cast as sqlalchemy_cast
from sqlalchemy import distinct, extract, func, text
from sqlalchemy.orm import Session
from sqlalchemy.sql.operators import in_op
from web3 import Web3
@ -54,6 +55,7 @@ subscription_id_by_blockchain = {
"wyrm": "wyrm_smartcontract",
"zksync_era_testnet": "zksync_era_testnet_smartcontract",
"zksync_era": "zksync_era_smartcontract",
"arbitrum_nova": "arbitrum_nova_smartcontract",
}
blockchain_by_subscription_id = {
@ -63,6 +65,7 @@ blockchain_by_subscription_id = {
"xdai_blockchain": "xdai",
"wyrm_blockchain": "wyrm",
"zksync_era_testnet_blockchain": "zksync_era_testnet",
"arbitrum_nova_blockchain": "arbitrum_nova",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"mumbai_smartcontract": "mumbai",
@ -70,6 +73,7 @@ blockchain_by_subscription_id = {
"wyrm_smartcontract": "wyrm",
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
"zksync_era_smartcontract": "zksync_era",
"arbitrum_nova_smartcontract": "arbitrum_nova",
}

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.3.5"
MOONCRAWL_VERSION = "0.3.6"

2
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -27,6 +27,8 @@ export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_mumb
export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_node>"
export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_testnet_node>"
export MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_node>"
export MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_arbitrum_nova_node>"
export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
# AWS environment variables

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -37,7 +37,7 @@ setup(
"bugout>=0.2.13",
"chardet",
"fastapi",
"moonstreamdb>=0.3.5",
"moonstreamdb>=0.3.6",
"moonstream>=0.1.1",
"moonworm[moonstream]>=0.6.2",
"humbug",