Amoy for mooncrawl

crawlers/mooncrawl/mooncrawl/blockchain.py

@@ -22,6 +22,7 @@ from .data import DateRange
 from .db import yield_db_session, yield_db_session_ctx
 from .settings import (
     MOONSTREAM_CRAWL_WORKERS,
+    MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI,
     MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI,
     MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI,
     MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI,
@@ -66,6 +67,8 @@ def connect(
             web3_uri = MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.MUMBAI:
             web3_uri = MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI
+        elif blockchain_type == AvailableBlockchainType.AMOY:
+            web3_uri = MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.XDAI:
             web3_uri = MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:

crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py

@@ -4,6 +4,7 @@ from typing import Optional
 from uuid import UUID
 
 from moonstreamdb.blockchain import AvailableBlockchainType
+from moonstreamdb.subscriptions import blockchain_type_to_subscription_type
 from web3 import Web3
 from web3.middleware import geth_poa_middleware
 
@@ -15,8 +16,6 @@ from ..settings import (
 )
 from .continuous_crawler import _retry_connect_web3, continuous_crawler
 from .crawler import (
-    SubscriptionTypes,
-    blockchain_type_to_subscription_type,
     find_all_deployed_blocks,
     get_crawl_job_entries,
     make_event_crawl_jobs,

crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py

@@ -6,7 +6,8 @@ from typing import Dict, List, Optional, Tuple
 from uuid import UUID
 
 from moonstreamdb.blockchain import AvailableBlockchainType
-from moonstreamdb.networks import Network
+from moonstreamdb.networks import blockchain_type_to_network_type
+from moonstreamdb.subscriptions import blockchain_type_to_subscription_type
 from moonworm.crawler.moonstream_ethereum_state_provider import (  # type: ignore
     MoonstreamEthereumStateProvider,
 )
@@ -21,7 +22,6 @@ from .crawler import (
     EventCrawlJob,
     FunctionCallCrawlJob,
     _retry_connect_web3,
-    blockchain_type_to_subscription_type,
     get_crawl_job_entries,
     heartbeat,
     make_event_crawl_jobs,
@@ -124,40 +124,10 @@ def continuous_crawler(
     if web3 is None:
         web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri)
 
-    if blockchain_type == AvailableBlockchainType.ETHEREUM:
+    try:
-        network = Network.ethereum
+        network = blockchain_type_to_network_type(blockchain_type=blockchain_type)
-    elif blockchain_type == AvailableBlockchainType.POLYGON:
+    except Exception as e:
-        network = Network.polygon
+        raise Exception(e)
-    elif blockchain_type == AvailableBlockchainType.MUMBAI:
-        network = Network.mumbai
-    elif blockchain_type == AvailableBlockchainType.XDAI:
-        network = Network.xdai
-    elif blockchain_type == AvailableBlockchainType.WYRM:
-        network = Network.wyrm
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
-        network = Network.zksync_era_testnet
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
-        network = Network.zksync_era
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
-        network = Network.zksync_era_sepolia
-    elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
-        network = Network.arbitrum_nova
-    elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
-        network = Network.arbitrum_sepolia
-    elif blockchain_type == AvailableBlockchainType.XAI:
-        network = Network.xai
-    elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
-        network = Network.xai_sepolia
-    elif blockchain_type == AvailableBlockchainType.AVALANCHE:
-        network = Network.avalanche
-    elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
-        network = Network.avalanche_fuji
-    elif blockchain_type == AvailableBlockchainType.BLAST:
-        network = Network.blast
-    elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
-        network = Network.blast_sepolia
-    else:
-        raise ValueError(f"Unknown blockchain type: {blockchain_type}")
 
     ethereum_state_provider = MoonstreamEthereumStateProvider(
         web3,

crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py

@@ -11,6 +11,7 @@ from uuid import UUID
 from bugout.data import BugoutJournalEntries, BugoutSearchResult
 from eth_typing.evm import ChecksumAddress
 from moonstreamdb.blockchain import AvailableBlockchainType
+from moonstreamdb.subscriptions import SubscriptionTypes
 from moonworm.deployment import find_deployment_block  # type: ignore
 from web3.main import Web3
 
@@ -29,25 +30,6 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
-class SubscriptionTypes(Enum):
-    POLYGON_BLOCKCHAIN = "polygon_smartcontract"
-    ETHEREUM_BLOCKCHAIN = "ethereum_smartcontract"
-    MUMBAI_BLOCKCHAIN = "mumbai_smartcontract"
-    XDAI_BLOCKCHAIN = "xdai_smartcontract"
-    WYRM_BLOCKCHAIN = "wyrm_smartcontract"
-    ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
-    ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract"
-    ZKSYNC_ERA_SEPOLIA_BLOCKCHAIN = "zksync_era_sepolia_smartcontract"
-    ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract"
-    ARBITRUM_SEPOLIA_BLOCKCHAIN = "arbitrum_sepolia_smartcontract"
-    XAI_BLOCKCHAIN = "xai_smartcontract"
-    XAI_SEPOLIA_BLOCKCHAIN = "xai_sepolia_smartcontract"
-    AVALANCHE_BLOCKCHAIN = "avalanche_smartcontract"
-    AVALANCHE_FUJI_BLOCKCHAIN = "avalanche_fuji_smartcontract"
-    BLAST_BLOCKCHAIN = "blast_smartcontract"
-    BLAST_SEPOLIA_BLOCKCHAIN = "blast_sepolia_smartcontract"
-
-
 def abi_input_signature(input_abi: Dict[str, Any]) -> str:
     """
     Stringifies a function ABI input object according to the ABI specification:
@@ -137,45 +119,6 @@ def _retry_connect_web3(
     )
 
 
-def blockchain_type_to_subscription_type(
-    blockchain_type: AvailableBlockchainType,
-) -> SubscriptionTypes:
-    if blockchain_type == AvailableBlockchainType.ETHEREUM:
-        return SubscriptionTypes.ETHEREUM_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.POLYGON:
-        return SubscriptionTypes.POLYGON_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.MUMBAI:
-        return SubscriptionTypes.MUMBAI_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.XDAI:
-        return SubscriptionTypes.XDAI_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.WYRM:
-        return SubscriptionTypes.WYRM_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
-        return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
-        return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
-        return SubscriptionTypes.ZKSYNC_ERA_SEPOLIA_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
-        return SubscriptionTypes.ARBITRUM_NOVA_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
-        return SubscriptionTypes.ARBITRUM_SEPOLIA_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.XAI:
-        return SubscriptionTypes.XAI_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
-        return SubscriptionTypes.XAI_SEPOLIA_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.AVALANCHE:
-        return SubscriptionTypes.AVALANCHE_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
-        return SubscriptionTypes.AVALANCHE_FUJI_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.BLAST:
-        return SubscriptionTypes.BLAST_BLOCKCHAIN
-    elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
-        return SubscriptionTypes.BLAST_SEPOLIA_BLOCKCHAIN
-    else:
-        raise ValueError(f"Unknown blockchain type: {blockchain_type}")
-
-
 @dataclass
 class EventCrawlJob:
     event_abi_hash: str

crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py

@@ -2,7 +2,7 @@ import logging
 from typing import List
 
 from moonstreamdb.blockchain import AvailableBlockchainType
-from moonstreamdb.networks import Network  # type: ignore
+from moonstreamdb.networks import blockchain_type_to_network_type  # type: ignore
 from moonworm.crawler.function_call_crawler import (  # type: ignore
     ContractFunctionCall,
     FunctionCallCrawler,
@@ -58,40 +58,10 @@ def function_call_crawler(
     end_block: int,
     batch_size: int,
 ):
-    if blockchain_type == AvailableBlockchainType.ETHEREUM:
+    try:
-        network = Network.ethereum
+        network = blockchain_type_to_network_type(blockchain_type=blockchain_type)
-    elif blockchain_type == AvailableBlockchainType.POLYGON:
+    except Exception as e:
-        network = Network.polygon
+        raise Exception(e)
-    elif blockchain_type == AvailableBlockchainType.MUMBAI:
-        network = Network.mumbai
-    elif blockchain_type == AvailableBlockchainType.XDAI:
-        network = Network.xdai
-    elif blockchain_type == AvailableBlockchainType.WYRM:
-        network = Network.wyrm
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
-        network = Network.zksync_era_testnet
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
-        network = Network.zksync_era
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
-        network = Network.zksync_era_sepolia
-    elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
-        network = Network.arbitrum_nova
-    elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
-        network = Network.arbitrum_sepolia
-    elif blockchain_type == AvailableBlockchainType.XAI:
-        network = Network.xai
-    elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
-        network = Network.xai_sepolia
-    elif blockchain_type == AvailableBlockchainType.AVALANCHE:
-        network = Network.avalanche
-    elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
-        network = Network.avalanche_fuji
-    elif blockchain_type == AvailableBlockchainType.BLAST:
-        network = Network.blast
-    elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
-        network = Network.blast_sepolia
-    else:
-        raise ValueError(f"Unknown blockchain type: {blockchain_type}")
 
     ethereum_state_provider = MoonstreamEthereumStateProvider(
         web3,

crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py

@@ -5,7 +5,7 @@ from uuid import UUID
 
 from eth_typing.evm import ChecksumAddress
 from moonstreamdb.blockchain import AvailableBlockchainType
-from moonstreamdb.networks import Network  # type: ignore
+from moonstreamdb.networks import blockchain_type_to_network_type  # type: ignore
 from moonworm.crawler.moonstream_ethereum_state_provider import (  # type: ignore
     MoonstreamEthereumStateProvider,
 )
@@ -52,40 +52,10 @@ def historical_crawler(
         web3.eth.block_number >= start_block
     ), "start_block must be less than current block"
 
-    if blockchain_type == AvailableBlockchainType.ETHEREUM:
+    try:
-        network = Network.ethereum
+        network = blockchain_type_to_network_type(blockchain_type=blockchain_type)
-    elif blockchain_type == AvailableBlockchainType.POLYGON:
+    except Exception as e:
-        network = Network.polygon
+        raise Exception(e)
-    elif blockchain_type == AvailableBlockchainType.MUMBAI:
-        network = Network.mumbai
-    elif blockchain_type == AvailableBlockchainType.XDAI:
-        network = Network.xdai
-    elif blockchain_type == AvailableBlockchainType.WYRM:
-        network = Network.wyrm
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
-        network = Network.zksync_era_testnet
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
-        network = Network.zksync_era
-    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
-        network = Network.zksync_era_sepolia
-    elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
-        network = Network.arbitrum_nova
-    elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
-        network = Network.arbitrum_sepolia
-    elif blockchain_type == AvailableBlockchainType.XAI:
-        network = Network.xai
-    elif blockchain_type == AvailableBlockchainType.XAI_SEPOLIA:
-        network = Network.xai_sepolia
-    elif blockchain_type == AvailableBlockchainType.AVALANCHE:
-        network = Network.avalanche
-    elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
-        network = Network.avalanche_fuji
-    elif blockchain_type == AvailableBlockchainType.BLAST:
-        network = Network.blast
-    elif blockchain_type == AvailableBlockchainType.BLAST_SEPOLIA:
-        network = Network.blast_sepolia
-    else:
-        raise Exception("Unsupported blockchain type provided")
 
     ethereum_state_provider = MoonstreamEthereumStateProvider(
         web3,

crawlers/mooncrawl/mooncrawl/settings.py

@@ -96,6 +96,12 @@ MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI = os.environ.get(
 if MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI == "":
     raise Exception("MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI env variable is not set")
 
+MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI = os.environ.get(
+    "MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI", ""
+)
+if MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI == "":
+    raise Exception("MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI env variable is not set")
+
 MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI = os.environ.get(
     "MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI", ""
 )

crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py

@@ -24,6 +24,10 @@ from moonstreamdb.blockchain import (
     get_label_model,
     get_transaction_model,
 )
+from moonstreamdb.subscriptions import (
+    blockchain_by_subscription_id,
+    subscription_id_by_blockchain,
+)
 from sqlalchemy import and_
 from sqlalchemy import cast as sqlalchemy_cast
 from sqlalchemy import distinct, extract, func, text
@@ -47,61 +51,6 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
-subscription_id_by_blockchain = {
-    "ethereum": "ethereum_smartcontract",
-    "polygon": "polygon_smartcontract",
-    "mumbai": "mumbai_smartcontract",
-    "xdai": "xdai_smartcontract",
-    "wyrm": "wyrm_smartcontract",
-    "zksync_era_testnet": "zksync_era_testnet_smartcontract",
-    "zksync_era": "zksync_era_smartcontract",
-    "zksync_era_sepolia": "zksync_era_sepolia_smartcontract",
-    "arbitrum_nova": "arbitrum_nova_smartcontract",
-    "arbitrum_sepolia": "arbitrum_sepolia_smartcontract",
-    "xai": "xai_smartcontract",
-    "xai_sepolia": "xai_sepolia_smartcontract",
-    "avalanche": "avalanche_smartcontract",
-    "avalanche_fuji": "avalanche_fuji_smartcontract",
-    "blast": "blast_smartcontract",
-    "blast_sepolia": "blast_sepolia_smartcontract",
-}
-
-blockchain_by_subscription_id = {
-    "ethereum_blockchain": "ethereum",
-    "polygon_blockchain": "polygon",
-    "mumbai_blockchain": "mumbai",
-    "xdai_blockchain": "xdai",
-    "wyrm_blockchain": "wyrm",
-    "zksync_era_testnet_blockchain": "zksync_era_testnet",
-    "zksync_era_blockchain": "zksync_era",
-    "zksync_era_sepolia_blockchain": "zksync_era_sepolia",
-    "arbitrum_nova_blockchain": "arbitrum_nova",
-    "arbitrum_sepolia_blockchain": "arbitrum_sepolia",
-    "xai_blockchain": "xai",
-    "xai_sepolia_blockchain": "xai_sepolia",
-    "avalanche_blockchain": "avalanche",
-    "avalanche_fuji_blockchain": "avalanche_fuji",
-    "blast_blockchain": "blast",
-    "blast_sepolia_blockchain": "blast_sepolia",
-    "ethereum_smartcontract": "ethereum",
-    "polygon_smartcontract": "polygon",
-    "mumbai_smartcontract": "mumbai",
-    "xdai_smartcontract": "xdai",
-    "wyrm_smartcontract": "wyrm",
-    "zksync_era_testnet_smartcontract": "zksync_era_testnet",
-    "zksync_era_smartcontract": "zksync_era",
-    "zksync_era_sepolia_smartcontract": "zksync_era_sepolia",
-    "arbitrum_nova_smartcontract": "arbitrum_nova",
-    "arbitrum_sepolia_smartcontract": "arbitrum_sepolia",
-    "xai_smartcontract": "xai",
-    "xai_sepolia_smartcontract": "xai_sepolia",
-    "avalanche_smartcontract": "avalanche",
-    "avalanche_fuji_smartcontract": "avalanche_fuji",
-    "blast_smartcontract": "blast",
-    "blast_sepolia_smartcontract": "blast_sepolia",
-}
-
-
 class TimeScale(Enum):
     # TODO(Andrey) Unlock when we be sure about perfomanse of agregation on transactions table.
     # Right now it can be hungs

crawlers/mooncrawl/mooncrawl/version.py

@@ -2,4 +2,4 @@
 Moonstream crawlers version.
 """
 
-MOONCRAWL_VERSION = "0.4.2"
+MOONCRAWL_VERSION = "0.4.3"

crawlers/mooncrawl/sample.env

@@ -24,6 +24,7 @@ export NFT_HUMBUG_TOKEN="<Token_for_nft_crawler>"
 export MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
+export MOONSTREAM_NODE_AMOY_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"

crawlers/mooncrawl/setup.py

@@ -37,7 +37,7 @@ setup(
         "bugout>=0.2.13",
         "chardet",
         "fastapi",
-        "moonstreamdb>=0.3.12",
+        "moonstreamdb>=0.4.1",
         "moonstream>=0.1.1",
         "moonworm[moonstream]>=0.6.2",
         "humbug",