access id default from env variable for all mooncrawl crawlers

crawlers/mooncrawl/mooncrawl/api.py

@@ -23,6 +23,7 @@ from .settings import (
     ORIGINS,
     bugout_client as bc,
     BUGOUT_RESOURCE_TYPE_SUBSCRIPTION,
+    NB_CONTROLLER_ACCESS_ID,
     MOONSTREAM_S3_QUERIES_BUCKET,
     MOONSTREAM_S3_QUERIES_BUCKET_PREFIX,
     MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
@@ -119,6 +120,7 @@ async def status_handler(
             timescales=stats_update.timescales,
             dashboard=dashboard_resource,
             subscription_by_id=subscription_by_id,
+            access_id=NB_CONTROLLER_ACCESS_ID,
         )
 
     except Exception as e:

crawlers/mooncrawl/mooncrawl/contract/cli.py

@@ -3,6 +3,7 @@ import json
 import logging
 import time
 from typing import Optional
+from uuid import UUID
 
 from moonstreamdb.db import yield_db_session_ctx
 from sqlalchemy.orm.session import Session
@@ -11,6 +12,7 @@ from web3 import Web3
 from ..blockchain import connect
 from ..data import AvailableBlockchainType
 from .deployment_crawler import ContractDeploymentCrawler, MoonstreamDataStore
+from ..settings import NB_CONTROLLER_ACCESS_ID
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -117,7 +119,7 @@ def run_crawler_desc(
 
 def handle_parser(args: argparse.Namespace):
     with yield_db_session_ctx() as session:
-        w3 = connect(AvailableBlockchainType.ETHEREUM)
+        w3 = connect(AvailableBlockchainType.ETHEREUM, access_id=args.access_id)
         if args.order == "asc":
             run_crawler_asc(
                 w3=w3,
@@ -153,6 +155,14 @@ def generate_parser():
     """
 
     parser = argparse.ArgumentParser(description="Moonstream Deployment Crawler")
+
+    parser.add_argument(
+        "--access-id",
+        default=NB_CONTROLLER_ACCESS_ID,
+        type=UUID,
+        help="User access ID",
+    )
+
     parser.add_argument(
         "--start", "-s", type=int, default=None, help="block to start crawling from"
     )

crawlers/mooncrawl/mooncrawl/crawler.py

@@ -23,7 +23,7 @@ from .blockchain import (
 )
 from .data import AvailableBlockchainType
 from .publish import publish_json
-from .settings import MOONSTREAM_CRAWL_WORKERS
+from .settings import NB_CONTROLLER_ACCESS_ID, MOONSTREAM_CRAWL_WORKERS
 from .version import MOONCRAWL_VERSION
 
 logging.basicConfig(level=logging.INFO)
@@ -257,6 +257,7 @@ def main() -> None:
 
     parser.add_argument(
         "--access-id",
+        default=NB_CONTROLLER_ACCESS_ID,
         type=UUID,
         help="User access ID",
     )

crawlers/mooncrawl/mooncrawl/nft/cli.py

@@ -8,6 +8,7 @@ import sys
 import time
 from datetime import datetime, timezone
 from typing import Any, Dict, Optional, cast
+from uuid import UUID
 
 from bugout.app import Bugout
 from bugout.journal import SearchOrder
@@ -20,6 +21,7 @@ from ..blockchain import connect
 from ..data import AvailableBlockchainType
 from ..publish import publish_json
 from ..settings import (
+    NB_CONTROLLER_ACCESS_ID,
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
     MOONSTREAM_DATA_JOURNAL_ID,
     MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
@@ -55,7 +57,11 @@ def web3_client_from_cli_or_env(args: argparse.Namespace) -> Web3:
         raise ValueError(
             "Could not find Web3 connection information in arguments or in MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI environment variable"
         )
-    return connect(AvailableBlockchainType.ETHEREUM, web3_connection_string)
+    return connect(
+        AvailableBlockchainType.ETHEREUM,
+        web3_connection_string,
+        access_id=args.access_id,
+    )
 
 
 def get_latest_block_from_node(web3_client: Web3):
@@ -256,6 +262,14 @@ def ethereum_summary_handler(args: argparse.Namespace) -> None:
 def main() -> None:
     parser = argparse.ArgumentParser(description="Moonstream NFT crawlers")
     parser.set_defaults(func=lambda _: parser.print_help())
+
+    parser.add_argument(
+        "--access-id",
+        default=NB_CONTROLLER_ACCESS_ID,
+        type=UUID,
+        help="User access ID",
+    )
+
     subcommands = parser.add_subparsers(description="Subcommands")
 
     parser_ethereum = subcommands.add_parser(

crawlers/mooncrawl/mooncrawl/settings.py

@@ -1,5 +1,6 @@
 import os
-from typing import cast
+from typing import cast, Optional
+from uuid import UUID
 
 from bugout.app import Bugout
 
@@ -104,3 +105,10 @@ NB_ACCESS_ID_HEADER = os.environ.get("NB_ACCESS_ID_HEADER", "x-node-balancer-acc
 NB_DATA_SOURCE_HEADER = os.environ.get(
     "NB_DATA_SOURCE_HEADER", "x-node-balancer-data-source"
 )
+
+NB_CONTROLLER_ACCESS_ID: Optional[UUID] = None
+NB_CONTROLLER_ACCESS_ID_RAW = os.environ.get("NB_CONTROLLER_ACCESS_ID", "")
+try:
+    NB_CONTROLLER_ACCESS_ID = UUID(NB_CONTROLLER_ACCESS_ID_RAW)
+except:
+    pass

crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py

@@ -8,7 +8,7 @@ import logging
 import time
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Optional
 from uuid import UUID
 
 import boto3  # type: ignore
@@ -28,6 +28,7 @@ from ..data import AvailableBlockchainType
 from ..reporter import reporter
 from ..settings import (
     CRAWLER_LABEL,
+    NB_CONTROLLER_ACCESS_ID,
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
     MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
 )
@@ -337,7 +338,9 @@ def generate_list_of_names(
 
 
 def process_external(
-    abi_external_calls: List[Dict[str, Any]], blockchain: AvailableBlockchainType
+    abi_external_calls: List[Dict[str, Any]],
+    blockchain: AvailableBlockchainType,
+    access_id: Optional[UUID] = None,
 ):
     """
     Request all required external data
@@ -383,7 +386,7 @@ def process_external(
             logger.error(f"Error processing external call: {e}")
 
     if external_calls:
-        web3_client = connect(blockchain)
+        web3_client = connect(blockchain, access_id=access_id)
 
     for extcall in external_calls:
         try:
@@ -434,6 +437,7 @@ def generate_web3_metrics(
     address: str,
     crawler_label: str,
     abi_json: Any,
+    access_id: Optional[UUID] = None,
 ) -> List[Any]:
     """
     Generate stats for cards components
@@ -446,6 +450,7 @@ def generate_web3_metrics(
     extention_data = process_external(
         abi_external_calls=abi_external_calls,
         blockchain=blockchain_type,
+        access_id=access_id,
     )
 
     extention_data.append(
@@ -611,6 +616,7 @@ def stats_generate_handler(args: argparse.Namespace):
                         address=address,
                         crawler_label=crawler_label,
                         abi_json=abi_json,
+                        access_id=args.access_id,
                     )
 
                     # Generate blocks state information
@@ -693,6 +699,7 @@ def stats_generate_api_task(
     timescales: List[str],
     dashboard: BugoutResource,
     subscription_by_id: Dict[str, BugoutResource],
+    access_id: Optional[UUID] = None,
 ):
     """
     Start crawler with generate.
@@ -770,6 +777,7 @@ def stats_generate_api_task(
                     address=address,
                     crawler_label=crawler_label,
                     abi_json=abi_json,
+                    access_id=access_id,
                 )
 
                 # Generate blocks state information
@@ -843,6 +851,14 @@ def stats_generate_api_task(
 def main() -> None:
     parser = argparse.ArgumentParser(description="Command Line Interface")
     parser.set_defaults(func=lambda _: parser.print_help())
+
+    parser.add_argument(
+        "--access-id",
+        default=NB_CONTROLLER_ACCESS_ID,
+        type=UUID,
+        help="User access ID",
+    )
+
     subcommands = parser.add_subparsers(
         description="Drone dashboard statistics commands"
     )

crawlers/mooncrawl/sample.env

@@ -1,20 +1,30 @@
+# Bugout environment variables
 export BUGOUT_BROOD_URL="https://auth.bugout.dev"
 export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
+export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout_Humbug_token_for_crash_reports>"
+
+# Moonstream environment variables
 export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to"
-export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_ethereum_node>"
-export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://<connection_path_uri_to_polygon_node>"
-export MOONSTREAM_CRAWL_WORKERS=4
 export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
 export MOONSTREAM_DB_URI_READ_ONLY="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
-export MOONSTREAM_ETHERSCAN_TOKEN="<Token_for_etherscan>"
-export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS_S3_bucket_for_smart_contracts>"
-export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx_for_AWS_S3_bucket_(prod,dev,..)>"
+export MOONSTREAM_CRAWL_WORKERS=4
 export MOONSTREAM_HUMBUG_TOKEN="<Token_for_crawlers_store_data_via_Humbug>"
-export COINMARKETCAP_API_KEY="<API_key_to_parse_conmarketcap>"
-export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout_Humbug_token_for_crash_reports>"
 export MOONSTREAM_DATA_JOURNAL_ID="<Bugout_journal_id_for_moonstream>"
+export MOONSTREAM_MOONWORM_TASKS_JOURNAL="<journal_with_tasks_for_moonworm_crawler>"
 export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Bugout_access_token_for_moonstream>"
 export NFT_HUMBUG_TOKEN="<Token_for_nft_crawler>"
-export MOONSTREAM_MOONWORM_TASKS_JOURNAL="<journal_with_tasks_for_moonworm_crawler>"
+
+# Blockchain nodes environment variables
+export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_ethereum_node>"
+export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://<connection_path_uri_to_polygon_node>"
+export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
+
+# AWS environment variables
+export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS_S3_bucket_for_smart_contracts>"
+export MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx_for_AWS_S3_bucket_(prod,dev,..)>"
 export MOONSTREAM_S3_QUERIES_BUCKET="<AWS_S3_bucket_to_store_sql_queries>"
 export MOONSTREAM_S3_QUERIES_BUCKET_PREFIX="dev"
+
+# 3rd parties environment variables
+export MOONSTREAM_ETHERSCAN_TOKEN="<Token_for_etherscan>"
+export COINMARKETCAP_API_KEY="<API_key_to_parse_conmarketcap>"