diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index c6243cee..65b65aac 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -3,17 +3,15 @@ from concurrent.futures import Future, ProcessPoolExecutor, ThreadPoolExecutor, from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union from uuid import UUID +from moonstream.backend import ( + AvailableBlockchainType, + get_block_model, + get_transaction_model, +) from moonstreamdb.db import yield_db_session, yield_db_session_ctx from moonstreamdb.models import ( EthereumBlock, - EthereumLabel, EthereumTransaction, - PolygonBlock, - PolygonLabel, - PolygonTransaction, - XDaiBlock, - XDaiLabel, - XDaiTransaction, ) from psycopg2.errors import UniqueViolation # type: ignore from sqlalchemy import Column, desc, func @@ -24,7 +22,7 @@ from web3 import HTTPProvider, IPCProvider, Web3 from web3.middleware import geth_poa_middleware from web3.types import BlockData -from .data import AvailableBlockchainType, DateRange +from .data import DateRange from .settings import ( MOONSTREAM_CRAWL_WORKERS, MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI, @@ -85,66 +83,6 @@ def connect( return web3_client -def get_block_model( - blockchain_type: AvailableBlockchainType, -) -> Type[Union[EthereumBlock, PolygonBlock]]: - """ - Depends on provided blockchain type: Ethereum or Polygon, - set proper blocks model: EthereumBlock or PolygonBlock. - """ - block_model: Type[Union[EthereumBlock, PolygonBlock]] - if blockchain_type == AvailableBlockchainType.ETHEREUM: - block_model = EthereumBlock - elif blockchain_type == AvailableBlockchainType.POLYGON: - block_model = PolygonBlock - elif blockchain_type == AvailableBlockchainType.XDAI: - block_model = XDaiBlock - else: - raise Exception("Unsupported blockchain type provided") - - return block_model - - -def get_label_model( - blockchain_type: AvailableBlockchainType, -) -> Type[Union[EthereumLabel, PolygonLabel]]: - """ - Depends on provided blockchain type: Ethereum or Polygon, - set proper block label model: EthereumLabel or PolygonLabel. - """ - label_model: Type[Union[EthereumLabel, PolygonLabel]] - if blockchain_type == AvailableBlockchainType.ETHEREUM: - label_model = EthereumLabel - elif blockchain_type == AvailableBlockchainType.POLYGON: - label_model = PolygonLabel - elif blockchain_type == AvailableBlockchainType.XDAI: - label_model = XDaiLabel - else: - raise Exception("Unsupported blockchain type provided") - - return label_model - - -def get_transaction_model( - blockchain_type: AvailableBlockchainType, -) -> Type[Union[EthereumTransaction, PolygonTransaction]]: - """ - Depends on provided blockchain type: Ethereum or Polygon, - set proper block transactions model: EthereumTransaction or PolygonTransaction. - """ - transaction_model: Type[Union[EthereumTransaction, PolygonTransaction]] - if blockchain_type == AvailableBlockchainType.ETHEREUM: - transaction_model = EthereumTransaction - elif blockchain_type == AvailableBlockchainType.POLYGON: - transaction_model = PolygonTransaction - elif blockchain_type == AvailableBlockchainType.XDAI: - transaction_model = XDaiTransaction - else: - raise Exception("Unsupported blockchain type provided") - - return transaction_model - - def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) -> None: """ Add block if doesn't presented in database. diff --git a/crawlers/mooncrawl/mooncrawl/contract/cli.py b/crawlers/mooncrawl/mooncrawl/contract/cli.py index cb77c31a..855cb097 100644 --- a/crawlers/mooncrawl/mooncrawl/contract/cli.py +++ b/crawlers/mooncrawl/mooncrawl/contract/cli.py @@ -1,16 +1,15 @@ import argparse -import json import logging import time from typing import Optional from uuid import UUID +from moonstream.backend import AvailableBlockchainType from moonstreamdb.db import yield_db_session_ctx from sqlalchemy.orm.session import Session from web3 import Web3 from ..blockchain import connect -from ..data import AvailableBlockchainType from ..settings import NB_CONTROLLER_ACCESS_ID from .deployment_crawler import ContractDeploymentCrawler, MoonstreamDataStore diff --git a/crawlers/mooncrawl/mooncrawl/crawler.py b/crawlers/mooncrawl/mooncrawl/crawler.py index a8ca13f0..3d6a870a 100644 --- a/crawlers/mooncrawl/mooncrawl/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/crawler.py @@ -12,6 +12,7 @@ from enum import Enum from typing import Iterator, List from uuid import UUID +from moonstream.backend import AvailableBlockchainType import dateutil.parser from .blockchain import ( @@ -21,7 +22,6 @@ from .blockchain import ( get_latest_blocks, trending, ) -from .data import AvailableBlockchainType from .publish import publish_json from .settings import MOONSTREAM_CRAWL_WORKERS, NB_CONTROLLER_ACCESS_ID from .version import MOONCRAWL_VERSION diff --git a/crawlers/mooncrawl/mooncrawl/data.py b/crawlers/mooncrawl/mooncrawl/data.py index 85c269e9..a3aa8587 100644 --- a/crawlers/mooncrawl/mooncrawl/data.py +++ b/crawlers/mooncrawl/mooncrawl/data.py @@ -6,12 +6,6 @@ from typing import Any, Dict, List from pydantic import BaseModel, Field -class AvailableBlockchainType(Enum): - ETHEREUM = "ethereum" - POLYGON = "polygon" - XDAI = "xdai" - - class StatsUpdateRequest(BaseModel): dashboard_id: str timescales: List[str] diff --git a/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py b/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py index d4a93d92..c610052d 100644 --- a/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py +++ b/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py @@ -4,15 +4,16 @@ import time from dataclasses import dataclass from typing import Any, Dict, List, Optional, Set, Union -import web3 from eth_typing import ChecksumAddress from hexbytes.main import HexBytes -from moonstreamdb.db import yield_db_session_ctx +from moonstream.backend import ( + AvailableBlockchainType, + get_label_model, + get_transaction_model, +) from moonstreamdb.models import ( Base, - EthereumLabel, EthereumTransaction, - PolygonLabel, PolygonTransaction, ) from moonworm.crawler.function_call_crawler import ( # type: ignore @@ -25,19 +26,9 @@ from tqdm import tqdm from web3 import Web3 from web3._utils.events import get_event_data -from mooncrawl.data import AvailableBlockchainType # type: ignore - -from ..blockchain import ( - connect, - get_block_model, - get_label_model, - get_transaction_model, -) from ..moonworm_crawler.db import ( - _event_to_label, add_events_to_session, commit_session, - get_last_labeled_block_number, ) from ..moonworm_crawler.event_crawler import Event, get_block_timestamp diff --git a/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py index 075698ec..09b78ee7 100644 --- a/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py @@ -4,12 +4,11 @@ import logging from typing import Optional from uuid import UUID +from moonstream.backend import AvailableBlockchainType from moonstreamdb.db import yield_db_session_ctx from web3 import Web3 from web3.middleware import geth_poa_middleware -from mooncrawl.data import AvailableBlockchainType # type: ignore - from ..blockchain import connect from ..settings import NB_CONTROLLER_ACCESS_ID from .base import crawl, get_checkpoint, populate_with_events diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py index 05128ec2..5f65a24c 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py @@ -3,11 +3,11 @@ import logging from typing import Optional from uuid import UUID +from moonstream.backend import AvailableBlockchainType from moonstreamdb.db import yield_db_session_ctx from web3 import Web3 from web3.middleware import geth_poa_middleware -from ..blockchain import AvailableBlockchainType from ..settings import MOONSTREAM_MOONWORM_TASKS_JOURNAL, NB_CONTROLLER_ACCESS_ID from .continuous_crawler import _retry_connect_web3, continuous_crawler from .crawler import ( diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index dc47bbff..45aab74c 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta from typing import Dict, List, Optional, Tuple from uuid import UUID +from moonstream.backend import AvailableBlockchainType from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore MoonstreamEthereumStateProvider, ) @@ -12,8 +13,6 @@ from moonworm.crawler.networks import Network # type: ignore from sqlalchemy.orm.session import Session from web3 import Web3 -from ..blockchain import connect -from ..data import AvailableBlockchainType from .crawler import ( EventCrawlJob, FunctionCallCrawlJob, diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index ce1cd48f..a9004a39 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -10,12 +10,9 @@ from uuid import UUID from bugout.data import BugoutSearchResult from eth_typing.evm import ChecksumAddress -from moonstreamdb.models import Base -from sqlalchemy.orm.session import Session +from moonstream.backend import AvailableBlockchainType from web3.main import Web3 -from mooncrawl.data import AvailableBlockchainType - from ..blockchain import connect from ..reporter import reporter from ..settings import ( diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py index 29c2f9f7..11cf27f4 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py @@ -1,12 +1,11 @@ import logging -from typing import Any, Dict, List, Optional, Union +from typing import Dict, List, Optional +from moonstream.backend import AvailableBlockchainType, get_label_model from moonstreamdb.models import Base from moonworm.crawler.function_call_crawler import ContractFunctionCall # type: ignore from sqlalchemy.orm import Session -from ..blockchain import get_label_model -from ..data import AvailableBlockchainType from ..settings import CRAWLER_LABEL from .event_crawler import Event diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py index 221dbc69..473d9da9 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py @@ -1,18 +1,13 @@ import logging -import traceback from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Union, cast +from typing import Any, Dict, List, Optional -from eth_typing.evm import ChecksumAddress -from moonstreamdb.models import Base +from moonstream.backend import AvailableBlockchainType, get_block_model, get_label_model from moonworm.crawler.log_scanner import _fetch_events_chunk # type: ignore from sqlalchemy.orm.session import Session from sqlalchemy.sql.expression import and_ from web3 import Web3 -from ..blockchain import connect, get_block_model, get_label_model -from ..data import AvailableBlockchainType -from ..settings import CRAWLER_LABEL from .crawler import EventCrawlJob logging.basicConfig(level=logging.INFO) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py index 3befe9a6..e81e90e9 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py @@ -1,16 +1,7 @@ import logging -from typing import Any, Dict, List, Optional, Union +from typing import List -from eth_typing.evm import ChecksumAddress -from hexbytes.main import HexBytes -from moonstreamdb.db import yield_db_session_ctx -from moonstreamdb.models import ( - Base, - EthereumLabel, - EthereumTransaction, - PolygonLabel, - PolygonTransaction, -) +from moonstream.backend import AvailableBlockchainType from moonworm.crawler.function_call_crawler import ( # type: ignore ContractFunctionCall, FunctionCallCrawler, @@ -23,9 +14,6 @@ from moonworm.cu_watch import MockState # type: ignore from sqlalchemy.orm import Session from web3 import Web3 -from ..blockchain import connect, get_block_model, get_label_model -from ..data import AvailableBlockchainType -from ..settings import CRAWLER_LABEL from .crawler import FunctionCallCrawlJob, _generate_reporter_callback logging.basicConfig(level=logging.INFO) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py index 2fb51928..22da8052 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py @@ -3,6 +3,7 @@ import time from typing import Dict, List, Optional, Tuple from uuid import UUID +from moonstream.backend import AvailableBlockchainType from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore MoonstreamEthereumStateProvider, ) @@ -10,7 +11,6 @@ from moonworm.crawler.networks import Network # type: ignore from sqlalchemy.orm.session import Session from web3 import Web3 -from ..data import AvailableBlockchainType from .crawler import EventCrawlJob, FunctionCallCrawlJob, _retry_connect_web3 from .db import add_events_to_session, add_function_calls_to_session, commit_session from .event_crawler import _crawl_events diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 421bd881..b87846a3 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -15,14 +15,18 @@ import traceback import boto3 # type: ignore from bugout.data import BugoutResource, BugoutResources +from moonstream.backend import ( + AvailableBlockchainType, + get_label_model, + get_transaction_model, +) from moonstreamdb.db import yield_db_read_only_session_ctx from sqlalchemy import and_, distinct, func, text, extract, cast from sqlalchemy.orm import Session from sqlalchemy.sql.operators import in_op from web3 import Web3 -from ..blockchain import connect, get_label_model, get_transaction_model -from ..data import AvailableBlockchainType +from ..blockchain import connect from ..reporter import reporter from ..settings import ( CRAWLER_LABEL, diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index a6c2d3a6..8b8237ab 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -37,7 +37,7 @@ setup( "bugout>=0.1.19", "chardet", "fastapi", - "moonstreamdb>=0.2.5", + "moonstreamdb>=0.3.1", "moonworm==0.2.4", "humbug", "pydantic",