From 249224df1decf29ae99f8c8de0dd5fb63b0c1762 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 25 Jun 2024 17:09:39 +0300 Subject: [PATCH 01/12] Add v3 crawler. --- .../mooncrawl/moonworm_crawler/cli.py | 13 ++- .../moonworm_crawler/continuous_crawler.py | 93 +++++++++++++++---- .../mooncrawl/moonworm_crawler/crawler.py | 10 +- 3 files changed, 92 insertions(+), 24 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py index a16bbfc0..8bab81db 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py @@ -197,7 +197,14 @@ def handle_crawl_v3(args: argparse.Namespace) -> None: index_engine = MoonstreamDBIndexesEngine() - with index_engine.yield_db_session_ctx() as index_db_session: + logger.info(f"Blockchain type: {blockchain_type.value}") + customer_connection = get_db_connection(args.customer_uuid) + + customer_engine = MoonstreamCustomDBEngine(customer_connection) + + index_engine = MoonstreamDBIndexesEngine() + + with customer_engine.yield_db_session_ctx() as db_session, index_engine.yield_db_session_ctx() as index_db_session: initial_event_jobs = get_event_crawl_job_records( index_db_session, @@ -219,8 +226,6 @@ def handle_crawl_v3(args: argparse.Namespace) -> None: f"Initial function call crawl jobs count: {len(initial_function_call_jobs)}" ) - logger.info(f"Blockchain type: {blockchain_type.value}") - with yield_db_session_ctx() as db_session: web3: Optional[Web3] = None if args.web3 is None: logger.info( @@ -292,6 +297,8 @@ def handle_crawl_v3(args: argparse.Namespace) -> None: args.heartbeat_interval, args.new_jobs_refetch_interval, web3_uri=args.web3_uri, + version=3, + index_db_session=index_db_session, ) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index c379c95a..dd504b33 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -3,7 +3,6 @@ import time import traceback from datetime import datetime, timedelta from typing import Dict, List, Optional, Tuple -from uuid import UUID from moonstreamtypes.blockchain import AvailableBlockchainType @@ -16,10 +15,6 @@ from moonworm.crawler.ethereum_state_provider import Web3StateProvider from sqlalchemy.orm.session import Session from web3 import Web3 -from ..settings import ( - HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES, - HISTORICAL_CRAWLER_STATUSES, -) from .crawler import ( EventCrawlJob, FunctionCallCrawlJob, @@ -31,10 +26,13 @@ from .crawler import ( merge_event_crawl_jobs, merge_function_call_crawl_jobs, moonworm_crawler_update_job_as_pickedup, + get_event_crawl_job_records, + get_function_call_crawl_job_records, ) from .db import add_events_to_session, add_function_calls_to_session, commit_session from .event_crawler import _crawl_events from .function_call_crawler import _crawl_functions +from ..settings import CRAWLER_LABEL, SEER_CRAWLER_LABEL logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -108,8 +106,16 @@ def continuous_crawler( web3_uri: Optional[str] = None, max_insert_batch: int = 10000, version: int = 2, + index_db_session: Optional[Session] = None, + customer_id: Optional[str] = None, ): crawler_type = "continuous" + if version == 3: + crawler_type = "continuous_v3" + + label_name = CRAWLER_LABEL + if version == 3: + label_name = SEER_CRAWLER_LABEL assert ( min_blocks_batch < max_blocks_batch ), "min_blocks_batch must be less than max_blocks_batch" @@ -191,6 +197,7 @@ def continuous_crawler( to_block=end_block, blocks_cache=blocks_cache, db_block_query_batch=min_blocks_batch * 2, + version=version, ) logger.info( f"Crawled {len(all_events)} events from {start_block} to {end_block}." @@ -203,9 +210,13 @@ def continuous_crawler( db_session, all_events[i : i + max_insert_batch], blockchain_type, + version, + label_name, ) else: - add_events_to_session(db_session, all_events, blockchain_type) + add_events_to_session( + db_session, all_events, blockchain_type, version, label_name + ) logger.info( f"Crawling function calls from {start_block} to {end_block}" @@ -228,10 +239,16 @@ def continuous_crawler( db_session, all_function_calls[i : i + max_insert_batch], blockchain_type, + db_version=version, + label_name=label_name, ) else: add_function_calls_to_session( - db_session, all_function_calls, blockchain_type + db_session, + all_function_calls, + blockchain_type, + db_version=version, + label_name=label_name, ) current_time = datetime.utcnow() @@ -239,20 +256,56 @@ def continuous_crawler( if current_time - jobs_refetchet_time > timedelta( seconds=new_jobs_refetch_interval ): - logger.info( - f"Refetching new jobs from bugout journal since {jobs_refetchet_time}" - ) - event_crawl_jobs, function_call_crawl_jobs = _refetch_new_jobs( - event_crawl_jobs, function_call_crawl_jobs, blockchain_type - ) + if version == 2: + ## Refetch new jobs from bugout journal + logger.info( + f"Refetching new jobs from bugout journal since {jobs_refetchet_time}" + ) + event_crawl_jobs, function_call_crawl_jobs = _refetch_new_jobs( + event_crawl_jobs, function_call_crawl_jobs, blockchain_type + ) - ( - event_crawl_jobs, - function_call_crawl_jobs, - ) = moonworm_crawler_update_job_as_pickedup( - event_crawl_jobs=event_crawl_jobs, - function_call_crawl_jobs=function_call_crawl_jobs, - ) + ( + event_crawl_jobs, + function_call_crawl_jobs, + ) = moonworm_crawler_update_job_as_pickedup( + event_crawl_jobs=event_crawl_jobs, + function_call_crawl_jobs=function_call_crawl_jobs, + ) + elif version == 3 and index_db_session is not None: + ## Refetch new jobs from index db + + updated_event_crawl_jobs = get_event_crawl_job_records( + index_db_session, + blockchain_type, + [], + {event.event_abi_hash: event for event in event_crawl_jobs}, + customer_id=customer_id, + ) + + event_crawl_jobs = [ + event for event in updated_event_crawl_jobs.values() + ] + + updated_function_call_crawl_jobs = ( + get_function_call_crawl_job_records( + index_db_session, + blockchain_type, + [], + { + function_call.contract_address: function_call + for function_call in function_call_crawl_jobs + }, + customer_id=customer_id, + ) + ) + + function_call_crawl_jobs = [ + function_call + for function_call in updated_function_call_crawl_jobs.values() + ] + else: + raise ValueError("Invalid version") jobs_refetchet_time = current_time diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index 79a4d58e..807e2ef3 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -704,6 +704,7 @@ def get_event_crawl_job_records( blockchain_type: AvailableBlockchainType, addresses: List[str], existing_crawl_job_records: Dict[str, EventCrawlJob], + customer_id: Optional[str] = None, ): """ Retrieve and update the event crawl job records from the database. @@ -712,9 +713,12 @@ def get_event_crawl_job_records( query = ( db_session.query(AbiJobs) .filter(AbiJobs.chain == blockchain_type.value) - .filter(func.length(AbiJobs.abi_selector) > 10) + .filter(func.cast(AbiJobs.abi_selector, JSON).op("->>")("type") == "event") ) + if customer_id is not None: + query = query.filter(AbiJobs.customer_id == customer_id) + if len(addresses) != 0: query = query.filter( AbiJobs.address.in_([binascii.unhexlify(address) for address in addresses]) @@ -770,6 +774,7 @@ def get_function_call_crawl_job_records( blockchain_type: AvailableBlockchainType, addresses: List[str], existing_crawl_job_records: Dict[str, FunctionCallCrawlJob], + customer_id: Optional[str] = None, ): """ Retrieve and update the function call crawl job records from the database. @@ -786,6 +791,9 @@ def get_function_call_crawl_job_records( ) ) + if customer_id is not None: + query = query.filter(AbiJobs.customer_id == customer_id) + if len(addresses) != 0: query = query.filter( AbiJobs.address.in_([binascii.unhexlify(address) for address in addresses]) From ec36ec6fed7f9893af80c1bf2b68871c56e01688 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 25 Jun 2024 14:17:38 +0000 Subject: [PATCH 02/12] Moonstream API mantle support --- moonstreamapi/configs/sample.env | 2 + .../moonstreamapi/admin/subscription_types.py | 44 +++++++++++++++++++ .../moonstreamapi/providers/__init__.py | 4 ++ .../providers/moonworm_provider.py | 16 +++++++ .../moonstreamapi/providers/transactions.py | 14 ++++++ moonstreamapi/moonstreamapi/settings.py | 13 ++++++ moonstreamapi/moonstreamapi/web3_provider.py | 8 +++- moonstreamapi/requirements.txt | 2 +- moonstreamapi/setup.py | 2 +- 9 files changed, 102 insertions(+), 3 deletions(-) diff --git a/moonstreamapi/configs/sample.env b/moonstreamapi/configs/sample.env index 824071a8..eae85085 100644 --- a/moonstreamapi/configs/sample.env +++ b/moonstreamapi/configs/sample.env @@ -34,6 +34,8 @@ export MOONSTREAM_AVALANCHE_WEB3_PROVIDER_URI="https://=0.2.15", "fastapi", "moonstream", - "moonstreamdb>=0.4.4", + "moonstreamdb>=0.4.5", "moonstreamdb-v3>=0.0.9", "humbug", "pydantic==1.10.2", From a6503c0c10b30535fb8b3ed116c0bf4f7cbd71f5 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 25 Jun 2024 14:19:03 +0000 Subject: [PATCH 03/12] Mantle support at engineapi --- engineapi/engineapi/settings.py | 2 ++ engineapi/sample.env | 2 ++ 2 files changed, 4 insertions(+) diff --git a/engineapi/engineapi/settings.py b/engineapi/engineapi/settings.py index 3d2450b6..a344c4d3 100644 --- a/engineapi/engineapi/settings.py +++ b/engineapi/engineapi/settings.py @@ -109,6 +109,8 @@ blockchain_names = [ "blast", "blast_sepolia", "proofofplay_apex", + "mantle", + "mantle_sepolia", ] for b in blockchain_names: diff --git a/engineapi/sample.env b/engineapi/sample.env index f24279e6..229da5f6 100644 --- a/engineapi/sample.env +++ b/engineapi/sample.env @@ -29,6 +29,8 @@ export MOONSTREAM_AVALANCHE_WEB3_PROVIDER_URI="https:// Date: Tue, 25 Jun 2024 17:25:25 +0300 Subject: [PATCH 04/12] Add changes --- crawlers/mooncrawl/mooncrawl/settings.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 0c796424..d5350a36 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -340,12 +340,24 @@ except: f"Could not parse WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS as int: {WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS_RAW}" ) - multicall_contracts: Dict[AvailableBlockchainType, str] = { AvailableBlockchainType.POLYGON: "0xc8E51042792d7405184DfCa245F2d27B94D013b6", + AvailableBlockchainType.AMOY: "0xcA11bde05977b3631167028862bE2a173976CA11", AvailableBlockchainType.MUMBAI: "0xe9939e7Ea7D7fb619Ac57f648Da7B1D425832631", AvailableBlockchainType.ETHEREUM: "0x5BA1e12693Dc8F9c48aAD8770482f4739bEeD696", + AvailableBlockchainType.SEPOLIA: "0xcA11bde05977b3631167028862bE2a173976CA11", AvailableBlockchainType.ZKSYNC_ERA: "0xF9cda624FBC7e059355ce98a31693d299FACd963", + AvailableBlockchainType.XDAI: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.XAI: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.BASE: "0xca11bde05977b3631167028862be2a173976ca11", + AvailableBlockchainType.ARBITRUM_ONE: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.ARBITRUM_NOVA: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.ARBITRUM_SEPOLIA: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.AVALANCHE: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.AVALANCHE_FUJI: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.BLAST: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.MANTLE: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.MANTLE_SEPOLIA: "0xcA11bde05977b3631167028862bE2a173976CA11", } From ec4b37662f55d174103d21bdf4f04db4f74ffd19 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 25 Jun 2024 14:35:14 +0000 Subject: [PATCH 05/12] Updated multicall contract addresses --- moonstreamapi/moonstreamapi/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/moonstreamapi/moonstreamapi/settings.py b/moonstreamapi/moonstreamapi/settings.py index b1269805..70e0109a 100644 --- a/moonstreamapi/moonstreamapi/settings.py +++ b/moonstreamapi/moonstreamapi/settings.py @@ -301,6 +301,11 @@ multicall_contracts: Dict[AvailableBlockchainType, str] = { AvailableBlockchainType.XDAI: "0xcA11bde05977b3631167028862bE2a173976CA11", AvailableBlockchainType.ZKSYNC_ERA: "0xF9cda624FBC7e059355ce98a31693d299FACd963", AvailableBlockchainType.ZKSYNC_ERA_TESTNET: "0xF9cda624FBC7e059355ce98a31693d299FACd963", + AvailableBlockchainType.AVALANCHE: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.AVALANCHE_FUJI: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.BLAST: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.MANTLE: "0xcA11bde05977b3631167028862bE2a173976CA11", + AvailableBlockchainType.MANTLE_SEPOLIA: "0xcA11bde05977b3631167028862bE2a173976CA11", } From 9c95e5c3d64835012f4dbd2ade47140a26ad84b7 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 25 Jun 2024 17:38:59 +0300 Subject: [PATCH 06/12] Add multicall. --- crawlers/mooncrawl/mooncrawl/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index d5350a36..4bcdaaed 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -347,6 +347,7 @@ multicall_contracts: Dict[AvailableBlockchainType, str] = { AvailableBlockchainType.ETHEREUM: "0x5BA1e12693Dc8F9c48aAD8770482f4739bEeD696", AvailableBlockchainType.SEPOLIA: "0xcA11bde05977b3631167028862bE2a173976CA11", AvailableBlockchainType.ZKSYNC_ERA: "0xF9cda624FBC7e059355ce98a31693d299FACd963", + AvailableBlockchainType.ZKSYNC_ERA_TESTNET: "0xF9cda624FBC7e059355ce98a31693d299FACd963", AvailableBlockchainType.XDAI: "0xcA11bde05977b3631167028862bE2a173976CA11", AvailableBlockchainType.XAI: "0xcA11bde05977b3631167028862bE2a173976CA11", AvailableBlockchainType.BASE: "0xca11bde05977b3631167028862be2a173976ca11", From df9996f7533e113cff2a6692f910434f4331d63c Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 25 Jun 2024 17:40:02 +0300 Subject: [PATCH 07/12] bump version --- crawlers/mooncrawl/mooncrawl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index dba8faaa..2ea34e26 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.4.8" +MOONCRAWL_VERSION = "0.4.9" From 1b239fa5bf925977ab71ca9d1c3a4d7bd43740c4 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 25 Jun 2024 19:03:25 +0300 Subject: [PATCH 08/12] Remove infura flag. --- crawlers/deploy/xai-sepolia-state.service | 2 +- crawlers/deploy/xai-state.service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crawlers/deploy/xai-sepolia-state.service b/crawlers/deploy/xai-sepolia-state.service index 27088dd1..19c14226 100644 --- a/crawlers/deploy/xai-sepolia-state.service +++ b/crawlers/deploy/xai-sepolia-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain xai_sepolia --infura +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain xai_sepolia CPUWeight=60 SyslogIdentifier=sepolia-state \ No newline at end of file diff --git a/crawlers/deploy/xai-state.service b/crawlers/deploy/xai-state.service index e3c835de..08cbbcfa 100644 --- a/crawlers/deploy/xai-state.service +++ b/crawlers/deploy/xai-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain xai --infura +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain xai CPUWeight=60 SyslogIdentifier=xai-state \ No newline at end of file From acd8b27effcaaa24ee3abf96aca066e370bf7321 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 26 Jun 2024 22:49:07 +0300 Subject: [PATCH 09/12] Add fix of customer_id filter. --- crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py | 9 +++++++++ crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py | 2 +- .../mooncrawl/moonworm_crawler/event_crawler.py | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py index 8bab81db..ca4e583f 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py @@ -211,6 +211,7 @@ def handle_crawl_v3(args: argparse.Namespace) -> None: blockchain_type, [], {}, + args.customer_uuid, ) logger.info(f"Initial event crawl jobs count: {len(initial_event_jobs)}") @@ -220,6 +221,7 @@ def handle_crawl_v3(args: argparse.Namespace) -> None: blockchain_type, [], {}, + args.customer_uuid, ) logger.info( @@ -878,6 +880,13 @@ def main() -> None: help="Force start from the start block", ) + crawl_parser_v3.add_argument( + "--customer-uuid", + type=UUID, + required=True, + help="Customer UUID", + ) + crawl_parser_v3.set_defaults(func=handle_crawl_v3) historical_crawl_parser = subparsers.add_parser( diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index 807e2ef3..6c266843 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -713,7 +713,7 @@ def get_event_crawl_job_records( query = ( db_session.query(AbiJobs) .filter(AbiJobs.chain == blockchain_type.value) - .filter(func.cast(AbiJobs.abi_selector, JSON).op("->>")("type") == "event") + .filter(func.cast(AbiJobs.abi, JSON).op("->>")("type") == "event") ) if customer_id is not None: diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py index af5bcd1f..f20e66ba 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py @@ -129,6 +129,7 @@ def _crawl_events( ) -> List[Event]: all_events = [] for job in jobs: + raw_events = _fetch_events_chunk( web3, job.event_abi, From 53c9726263d68887b9cccb1eaf7f7cc3f92f259b Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 27 Jun 2024 11:28:29 +0000 Subject: [PATCH 10/12] Raise var value error fix for mdb v3 --- crawlers/mooncrawl/setup.py | 2 +- moonstreamdb-v3/moonstreamdbv3/db.py | 35 ++++++++++------------ moonstreamdb-v3/moonstreamdbv3/version.txt | 2 +- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 19cf9ad5..90f35ee0 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -38,7 +38,7 @@ setup( "chardet", "fastapi", "moonstreamdb>=0.4.4", - "moonstreamdb-v3>=0.0.10", + "moonstreamdb-v3>=0.0.11", "moonstream-types>=0.0.3", "moonstream>=0.1.1", "moonworm[moonstream]>=0.9.2", diff --git a/moonstreamdb-v3/moonstreamdbv3/db.py b/moonstreamdb-v3/moonstreamdbv3/db.py index 92563115..0860b42b 100644 --- a/moonstreamdb-v3/moonstreamdbv3/db.py +++ b/moonstreamdb-v3/moonstreamdbv3/db.py @@ -13,30 +13,25 @@ from sqlalchemy.orm import Session, sessionmaker logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -try: - MOONSTREAM_DB_V3_URI = os.environ.get("MOONSTREAM_DB_V3_URI") - if MOONSTREAM_DB_V3_URI is None: - raise Warning("MOONSTREAM_DB_V3_URI environment variable must be set") +MOONSTREAM_DB_V3_URI = os.environ.get("MOONSTREAM_DB_V3_URI") +if MOONSTREAM_DB_V3_URI is None: + logger.warning("MOONSTREAM_DB_V3_URI environment variable must be set") - MOONSTREAM_DB_V3_URI_READ_ONLY = os.environ.get("MOONSTREAM_DB_V3_URI_READ_ONLY") - if MOONSTREAM_DB_V3_URI_READ_ONLY is None: - raise Warning("MOONSTREAM_DB_V3_URI_READ_ONLY environment variable must be set") +MOONSTREAM_DB_V3_URI_READ_ONLY = os.environ.get("MOONSTREAM_DB_V3_URI_READ_ONLY") +if MOONSTREAM_DB_V3_URI_READ_ONLY is None: + logger.warning("MOONSTREAM_DB_V3_URI_READ_ONLY environment variable must be set") - MOONSTREAM_DB_V3_INDEXES_URI = os.environ.get("MOONSTREAM_DB_V3_INDEXES_URI") - if MOONSTREAM_DB_V3_INDEXES_URI is None: - raise Warning("MOONSTREAM_DB_V3_INDEXES_URI environment variable must be set") +MOONSTREAM_DB_V3_INDEXES_URI = os.environ.get("MOONSTREAM_DB_V3_INDEXES_URI") +if MOONSTREAM_DB_V3_INDEXES_URI is None: + raise ValueError("MOONSTREAM_DB_V3_INDEXES_URI environment variable must be set") - MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY = os.environ.get( - "MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY" +MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY = os.environ.get( + "MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY" +) +if MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY is None: + raise ValueError( + "MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY environment variable must be set" ) - if MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY is None: - raise Warning( - "MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY environment variable must be set" - ) -except ValueError as e: - raise ValueError(e) -except Warning: - logger.warning("Database variables not set") MOONSTREAM_POOL_SIZE_RAW = os.environ.get("MOONSTREAM_POOL_SIZE") MOONSTREAM_POOL_SIZE = 1 diff --git a/moonstreamdb-v3/moonstreamdbv3/version.txt b/moonstreamdb-v3/moonstreamdbv3/version.txt index 7c1886bb..2cfabea2 100644 --- a/moonstreamdb-v3/moonstreamdbv3/version.txt +++ b/moonstreamdb-v3/moonstreamdbv3/version.txt @@ -1 +1 @@ -0.0.10 +0.0.11 From fa176ec2044df1225cf1e049fdbd5bcfaf7068de Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 27 Jun 2024 11:31:01 +0000 Subject: [PATCH 11/12] Fix --- moonstreamdb-v3/moonstreamdbv3/db.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/moonstreamdb-v3/moonstreamdbv3/db.py b/moonstreamdb-v3/moonstreamdbv3/db.py index 0860b42b..14a1e02b 100644 --- a/moonstreamdb-v3/moonstreamdbv3/db.py +++ b/moonstreamdb-v3/moonstreamdbv3/db.py @@ -21,17 +21,22 @@ MOONSTREAM_DB_V3_URI_READ_ONLY = os.environ.get("MOONSTREAM_DB_V3_URI_READ_ONLY" if MOONSTREAM_DB_V3_URI_READ_ONLY is None: logger.warning("MOONSTREAM_DB_V3_URI_READ_ONLY environment variable must be set") -MOONSTREAM_DB_V3_INDEXES_URI = os.environ.get("MOONSTREAM_DB_V3_INDEXES_URI") -if MOONSTREAM_DB_V3_INDEXES_URI is None: - raise ValueError("MOONSTREAM_DB_V3_INDEXES_URI environment variable must be set") +try: + MOONSTREAM_DB_V3_INDEXES_URI = os.environ.get("MOONSTREAM_DB_V3_INDEXES_URI") + if MOONSTREAM_DB_V3_INDEXES_URI is None: + raise Warning("MOONSTREAM_DB_V3_INDEXES_URI environment variable must be set") -MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY = os.environ.get( - "MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY" -) -if MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY is None: - raise ValueError( - "MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY environment variable must be set" + MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY = os.environ.get( + "MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY" ) + if MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY is None: + raise Warning( + "MOONSTREAM_DB_V3_INDEXES_URI_READ_ONLY environment variable must be set" + ) +except ValueError as e: + raise ValueError(e) +except Warning: + logger.warning("Indexes database variables not set") MOONSTREAM_POOL_SIZE_RAW = os.environ.get("MOONSTREAM_POOL_SIZE") MOONSTREAM_POOL_SIZE = 1 From 6c2cf33c719aef9f3eb0f14c75ab789b474993e6 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 27 Jun 2024 15:03:58 +0300 Subject: [PATCH 12/12] Add version parameter. --- crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py index ca4e583f..32af00d8 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py @@ -245,7 +245,9 @@ def handle_crawl_v3(args: argparse.Namespace) -> None: logger.info("Using PoA middleware") web3.middleware_onion.inject(geth_poa_middleware, layer=0) - last_labeled_block = get_last_labeled_block_number(db_session, blockchain_type) + last_labeled_block = get_last_labeled_block_number( + db_session, blockchain_type, db_version=3 + ) logger.info(f"Last labeled block: {last_labeled_block}") start_block = args.start