From 49454a87cc96f34ee746c99f76111eaa2461681e Mon Sep 17 00:00:00 2001 From: yhtiyar Date: Thu, 16 Dec 2021 22:46:44 +0300 Subject: [PATCH] isort and mypy fixes --- .../mooncrawl/moonworm_crawler/cli.py | 13 +++---- .../moonworm_crawler/continuous_crawler.py | 9 ++--- .../mooncrawl/moonworm_crawler/crawler.py | 37 ------------------- .../mooncrawl/moonworm_crawler/db.py | 5 +-- .../moonworm_crawler/function_call_crawler.py | 8 ++-- 5 files changed, 15 insertions(+), 57 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py index df6554a6..7c21e13a 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py @@ -1,20 +1,19 @@ -import logging import argparse +import logging -from web3 import Web3 from moonstreamdb.db import yield_db_session_ctx +from web3 import Web3 from web3.middleware import geth_poa_middleware +from ..blockchain import AvailableBlockchainType from ..settings import MOONSTREAM_MOONWORM_TASKS_JOURNAL, bugout_client +from .continuous_crawler import continuous_crawler from .crawler import ( + SubscriptionTypes, + get_crawl_job_entries, make_event_crawl_jobs, make_function_call_crawl_jobs, - get_crawl_job_entries, - SubscriptionTypes, ) -from ..blockchain import AvailableBlockchainType -from .continuous_crawler import continuous_crawler - logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index be6ec4f4..7a60b389 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -4,13 +4,13 @@ import traceback from datetime import datetime, timedelta from typing import Dict, List, Optional, Tuple -from moonworm.crawler.moonstream_ethereum_state_provider import ( +from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore MoonstreamEthereumStateProvider, ) -from moonworm.crawler.networks import Network +from moonworm.crawler.networks import Network # type: ignore from sqlalchemy.orm.session import Session -from sqlalchemy.sql.expression import and_ from web3 import Web3 + from ..blockchain import connect from ..data import AvailableBlockchainType from .crawler import ( @@ -24,10 +24,9 @@ from .crawler import ( merge_event_crawl_jobs, merge_function_call_crawl_jobs, ) - +from .db import save_events, save_function_calls from .event_crawler import _crawl_events from .function_call_crawler import _crawl_functions -from .db import save_events, save_function_calls logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index 1a1d1043..0e36d276 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -5,7 +5,6 @@ import time from dataclasses import dataclass from datetime import datetime from enum import Enum -from os import SEEK_CUR from typing import Any, Callable, Dict, List, Optional, cast from bugout.data import BugoutSearchResult @@ -16,7 +15,6 @@ from web3.main import Web3 from mooncrawl.data import AvailableBlockchainType -from ..blockchain import connect, get_block_model, get_label_model from ..reporter import reporter from ..settings import ( CRAWLER_LABEL, @@ -78,33 +76,6 @@ class FunctionCallCrawlJob: created_at: int -def _retry_connect_web3( - blockchain_type: AvailableBlockchainType, - retry_count: int = 10, - sleep_time: float = 5, -) -> Web3: - """ - Retry connecting to the blockchain. - """ - while retry_count > 0: - retry_count -= 1 - try: - web3 = connect(blockchain_type) - web3.eth.block_number - logger.info(f"Connected to {blockchain_type}") - return web3 - except Exception as e: - if retry_count == 0: - error = e - break - logger.error(f"Failed to connect to {blockchain_type} blockchain: {e}") - logger.info(f"Retrying in {sleep_time} seconds") - time.sleep(sleep_time) - raise Exception( - f"Failed to connect to {blockchain_type} blockchain after {retry_count} retries: {error}" - ) - - def get_crawl_job_entries( subscription_type: SubscriptionTypes, crawler_type: str, @@ -313,11 +284,3 @@ def heartbeat( entry_id=heartbeat_entry_id, tags=[crawler_type, "heartbeat", blockchain_type.value, "dead"], ) - - - - - - - - diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py index 5e2e9b5d..ebdda0be 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py @@ -11,10 +11,7 @@ from moonstreamdb.models import ( PolygonLabel, PolygonTransaction, ) -from moonworm.crawler.function_call_crawler import ( - ContractFunctionCall, -) - +from moonworm.crawler.function_call_crawler import ContractFunctionCall # type: ignore from sqlalchemy.orm import Session from sqlalchemy.sql.expression import label diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py index 926586ad..a98e057f 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py @@ -11,15 +11,15 @@ from moonstreamdb.models import ( PolygonLabel, PolygonTransaction, ) -from moonworm.crawler.function_call_crawler import ( +from moonworm.crawler.function_call_crawler import ( # type: ignore ContractFunctionCall, FunctionCallCrawler, ) -from moonworm.crawler.moonstream_ethereum_state_provider import ( +from moonworm.crawler.moonstream_ethereum_state_provider import ( # type: ignore MoonstreamEthereumStateProvider, ) -from moonworm.crawler.networks import Network -from moonworm.cu_watch import MockState +from moonworm.crawler.networks import Network # type: ignore +from moonworm.cu_watch import MockState # type: ignore from sqlalchemy.orm import Session from web3 import Web3