Modifications of crawlers for mumbai

crawlers/mooncrawl/mooncrawl/blockchain.py

@@ -3,17 +3,15 @@ from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor,
 from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
 from uuid import UUID
 
+from moonstream.backend import (
+    AvailableBlockchainType,
+    get_block_model,
+    get_transaction_model,
+)
 from moonstreamdb.db import yield_db_session, yield_db_session_ctx
 from moonstreamdb.models import (
     EthereumBlock,
-    EthereumLabel,
     EthereumTransaction,
-    PolygonBlock,
-    PolygonLabel,
-    PolygonTransaction,
-    XDaiBlock,
-    XDaiLabel,
-    XDaiTransaction,
 )
 from psycopg2.errors import UniqueViolation  # type: ignore
 from sqlalchemy import Column, desc, func
@@ -24,7 +22,7 @@ from web3 import HTTPProvider, IPCProvider, Web3
 from web3.middleware import geth_poa_middleware
 from web3.types import BlockData
 
-from .data import AvailableBlockchainType, DateRange
+from .data import DateRange
 from .settings import (
     MOONSTREAM_CRAWL_WORKERS,
     MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
@@ -85,66 +83,6 @@ def connect(
     return web3_client
 
 
-def get_block_model(
-    blockchain_type: AvailableBlockchainType,
-) -> Type[Union[EthereumBlock, PolygonBlock]]:
-    """
-    Depends on provided blockchain type: Ethereum or Polygon,
-    set proper blocks model: EthereumBlock or PolygonBlock.
-    """
-    block_model: Type[Union[EthereumBlock, PolygonBlock]]
-    if blockchain_type == AvailableBlockchainType.ETHEREUM:
-        block_model = EthereumBlock
-    elif blockchain_type == AvailableBlockchainType.POLYGON:
-        block_model = PolygonBlock
-    elif blockchain_type == AvailableBlockchainType.XDAI:
-        block_model = XDaiBlock
-    else:
-        raise Exception("Unsupported blockchain type provided")
-
-    return block_model
-
-
-def get_label_model(
-    blockchain_type: AvailableBlockchainType,
-) -> Type[Union[EthereumLabel, PolygonLabel]]:
-    """
-    Depends on provided blockchain type: Ethereum or Polygon,
-    set proper block label model: EthereumLabel or PolygonLabel.
-    """
-    label_model: Type[Union[EthereumLabel, PolygonLabel]]
-    if blockchain_type == AvailableBlockchainType.ETHEREUM:
-        label_model = EthereumLabel
-    elif blockchain_type == AvailableBlockchainType.POLYGON:
-        label_model = PolygonLabel
-    elif blockchain_type == AvailableBlockchainType.XDAI:
-        label_model = XDaiLabel
-    else:
-        raise Exception("Unsupported blockchain type provided")
-
-    return label_model
-
-
-def get_transaction_model(
-    blockchain_type: AvailableBlockchainType,
-) -> Type[Union[EthereumTransaction, PolygonTransaction]]:
-    """
-    Depends on provided blockchain type: Ethereum or Polygon,
-    set proper block transactions model: EthereumTransaction or PolygonTransaction.
-    """
-    transaction_model: Type[Union[EthereumTransaction, PolygonTransaction]]
-    if blockchain_type == AvailableBlockchainType.ETHEREUM:
-        transaction_model = EthereumTransaction
-    elif blockchain_type == AvailableBlockchainType.POLYGON:
-        transaction_model = PolygonTransaction
-    elif blockchain_type == AvailableBlockchainType.XDAI:
-        transaction_model = XDaiTransaction
-    else:
-        raise Exception("Unsupported blockchain type provided")
-
-    return transaction_model
-
-
 def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) -> None:
     """
     Add block if doesn't presented in database.

crawlers/mooncrawl/mooncrawl/contract/cli.py

@@ -1,16 +1,15 @@
 import argparse
-import json
 import logging
 import time
 from typing import Optional
 from uuid import UUID
 
+from moonstream.backend import AvailableBlockchainType
 from moonstreamdb.db import yield_db_session_ctx
 from sqlalchemy.orm.session import Session
 from web3 import Web3
 
 from ..blockchain import connect
-from ..data import AvailableBlockchainType
 from ..settings import NB_CONTROLLER_ACCESS_ID
 from .deployment_crawler import ContractDeploymentCrawler, MoonstreamDataStore
 

crawlers/mooncrawl/mooncrawl/crawler.py

@@ -12,6 +12,7 @@ from enum import Enum
 from typing import Iterator, List
 from uuid import UUID
 
+from moonstream.backend import AvailableBlockchainType
 import dateutil.parser
 
 from .blockchain import (
@@ -21,7 +22,6 @@ from .blockchain import (
     get_latest_blocks,
     trending,
 )
-from .data import AvailableBlockchainType
 from .publish import publish_json
 from .settings import MOONSTREAM_CRAWL_WORKERS, NB_CONTROLLER_ACCESS_ID
 from .version import MOONCRAWL_VERSION

crawlers/mooncrawl/mooncrawl/data.py

@@ -6,12 +6,6 @@ from typing import Any, Dict, List
 from pydantic import BaseModel, Field
 
 
-class AvailableBlockchainType(Enum):
-    ETHEREUM = "ethereum"
-    POLYGON = "polygon"
-    XDAI = "xdai"
-
-
 class StatsUpdateRequest(BaseModel):
     dashboard_id: str
     timescales: List[str]

crawlers/mooncrawl/mooncrawl/generic_crawler/base.py

@@ -4,15 +4,16 @@ import time
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Set, Union
 
-import web3
 from eth_typing import ChecksumAddress
 from hexbytes.main import HexBytes
-from moonstreamdb.db import yield_db_session_ctx
+from moonstream.backend import (
+    AvailableBlockchainType,
+    get_label_model,
+    get_transaction_model,
+)
 from moonstreamdb.models import (
     Base,
-    EthereumLabel,
     EthereumTransaction,
-    PolygonLabel,
     PolygonTransaction,
 )
 from moonworm.crawler.function_call_crawler import (  # type: ignore
@@ -25,19 +26,9 @@ from tqdm import tqdm
 from web3 import Web3
 from web3._utils.events import get_event_data
 
-from mooncrawl.data import AvailableBlockchainType  # type: ignore
-
-from ..blockchain import (
-    connect,
-    get_block_model,
-    get_label_model,
-    get_transaction_model,
-)
 from ..moonworm_crawler.db import (
-    _event_to_label,
     add_events_to_session,
     commit_session,
-    get_last_labeled_block_number,
 )
 from ..moonworm_crawler.event_crawler import Event, get_block_timestamp
 

crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py

@@ -4,12 +4,11 @@ import logging
 from typing import Optional
 from uuid import UUID
 
+from moonstream.backend import AvailableBlockchainType
 from moonstreamdb.db import yield_db_session_ctx
 from web3 import Web3
 from web3.middleware import geth_poa_middleware
 
-from mooncrawl.data import AvailableBlockchainType  # type: ignore
-
 from ..blockchain import connect
 from ..settings import NB_CONTROLLER_ACCESS_ID
 from .base import crawl, get_checkpoint, populate_with_events

crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py

@@ -3,11 +3,11 @@ import logging
 from typing import Optional
 from uuid import UUID
 
+from moonstream.backend import AvailableBlockchainType
 from moonstreamdb.db import yield_db_session_ctx
 from web3 import Web3
 from web3.middleware import geth_poa_middleware
 
-from ..blockchain import AvailableBlockchainType
 from ..settings import MOONSTREAM_MOONWORM_TASKS_JOURNAL, NB_CONTROLLER_ACCESS_ID
 from .continuous_crawler import _retry_connect_web3, continuous_crawler
 from .crawler import (

crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py

@@ -5,6 +5,7 @@ from datetime import datetime, timedelta
 from typing import Dict, List, Optional, Tuple
 from uuid import UUID
 
+from moonstream.backend import AvailableBlockchainType
 from moonworm.crawler.moonstream_ethereum_state_provider import (  # type: ignore
     MoonstreamEthereumStateProvider,
 )
@@ -12,8 +13,6 @@ from moonworm.crawler.networks import Network  # type: ignore
 from sqlalchemy.orm.session import Session
 from web3 import Web3
 
-from ..blockchain import connect
-from ..data import AvailableBlockchainType
 from .crawler import (
     EventCrawlJob,
     FunctionCallCrawlJob,

crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py

@@ -10,12 +10,9 @@ from uuid import UUID
 
 from bugout.data import BugoutSearchResult
 from eth_typing.evm import ChecksumAddress
-from moonstreamdb.models import Base
-from sqlalchemy.orm.session import Session
+from moonstream.backend import AvailableBlockchainType
 from web3.main import Web3
 
-from mooncrawl.data import AvailableBlockchainType
-
 from ..blockchain import connect
 from ..reporter import reporter
 from ..settings import (

crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py

@@ -1,12 +1,11 @@
 import logging
-from typing import Any, Dict, List, Optional, Union
+from typing import Dict, List, Optional
 
+from moonstream.backend import AvailableBlockchainType, get_label_model
 from moonstreamdb.models import Base
 from moonworm.crawler.function_call_crawler import ContractFunctionCall  # type: ignore
 from sqlalchemy.orm import Session
 
-from ..blockchain import get_label_model
-from ..data import AvailableBlockchainType
 from ..settings import CRAWLER_LABEL
 from .event_crawler import Event
 

crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py

@@ -1,18 +1,13 @@
 import logging
-import traceback
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Union, cast
+from typing import Any, Dict, List, Optional
 
-from eth_typing.evm import ChecksumAddress
-from moonstreamdb.models import Base
+from moonstream.backend import AvailableBlockchainType, get_block_model, get_label_model
 from moonworm.crawler.log_scanner import _fetch_events_chunk  # type: ignore
 from sqlalchemy.orm.session import Session
 from sqlalchemy.sql.expression import and_
 from web3 import Web3
 
-from ..blockchain import connect, get_block_model, get_label_model
-from ..data import AvailableBlockchainType
-from ..settings import CRAWLER_LABEL
 from .crawler import EventCrawlJob
 
 logging.basicConfig(level=logging.INFO)

crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py

@@ -1,16 +1,7 @@
 import logging
-from typing import Any, Dict, List, Optional, Union
+from typing import List
 
-from eth_typing.evm import ChecksumAddress
-from hexbytes.main import HexBytes
-from moonstreamdb.db import yield_db_session_ctx
-from moonstreamdb.models import (
-    Base,
-    EthereumLabel,
-    EthereumTransaction,
-    PolygonLabel,
-    PolygonTransaction,
-)
+from moonstream.backend import AvailableBlockchainType
 from moonworm.crawler.function_call_crawler import (  # type: ignore
     ContractFunctionCall,
     FunctionCallCrawler,
@@ -23,9 +14,6 @@ from moonworm.cu_watch import MockState  # type: ignore
 from sqlalchemy.orm import Session
 from web3 import Web3
 
-from ..blockchain import connect, get_block_model, get_label_model
-from ..data import AvailableBlockchainType
-from ..settings import CRAWLER_LABEL
 from .crawler import FunctionCallCrawlJob, _generate_reporter_callback
 
 logging.basicConfig(level=logging.INFO)

crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py

@@ -3,6 +3,7 @@ import time
 from typing import Dict, List, Optional, Tuple
 from uuid import UUID
 
+from moonstream.backend import AvailableBlockchainType
 from moonworm.crawler.moonstream_ethereum_state_provider import (  # type: ignore
     MoonstreamEthereumStateProvider,
 )
@@ -10,7 +11,6 @@ from moonworm.crawler.networks import Network  # type: ignore
 from sqlalchemy.orm.session import Session
 from web3 import Web3
 
-from ..data import AvailableBlockchainType
 from .crawler import EventCrawlJob, FunctionCallCrawlJob, _retry_connect_web3
 from .db import add_events_to_session, add_function_calls_to_session, commit_session
 from .event_crawler import _crawl_events

crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py

@@ -15,14 +15,18 @@ import traceback
 
 import boto3  # type: ignore
 from bugout.data import BugoutResource, BugoutResources
+from moonstream.backend import (
+    AvailableBlockchainType,
+    get_label_model,
+    get_transaction_model,
+)
 from moonstreamdb.db import yield_db_read_only_session_ctx
 from sqlalchemy import and_, distinct, func, text, extract, cast
 from sqlalchemy.orm import Session
 from sqlalchemy.sql.operators import in_op
 from web3 import Web3
 
-from ..blockchain import connect, get_label_model, get_transaction_model
-from ..data import AvailableBlockchainType
+from ..blockchain import connect
 from ..reporter import reporter
 from ..settings import (
     CRAWLER_LABEL,

crawlers/mooncrawl/setup.py

@@ -37,7 +37,7 @@ setup(
         "bugout>=0.1.19",
         "chardet",
         "fastapi",
-        "moonstreamdb>=0.2.5",
+        "moonstreamdb>=0.3.1",
         "moonworm==0.2.4",
         "humbug",
         "pydantic",