Add changes.

crawlers/deploy/leaderboards-worker.service

@@ -8,4 +8,4 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.leaderboards_generator.cli leaderboards-generate --query-api-access-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}"
 CPUWeight=60
-SyslogIdentifier=leaderboards-generator
+SyslogIdentifier=leaderboards-worker

crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py

@@ -13,6 +13,8 @@ from ..settings import (
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
     MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
     BUGOUT_REQUEST_TIMEOUT_SECONDS,
+    MOONSTREAM_API_URL,
+    MOONSTREAM_ENGINE_URL,
 )
 
 from ..settings import bugout_client as bc
@@ -21,6 +23,10 @@ from ..settings import bugout_client as bc
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
+blue_c = "\033[94m"
+green_c = "\033[92m"
+end_c = "\033[0m"
+
 
 def handle_leaderboards(args: argparse.Namespace) -> None:
     """
@@ -33,19 +39,23 @@ def handle_leaderboards(args: argparse.Namespace) -> None:
 
     query = "#leaderboard"
 
-    if args.leaderboard_id:
+    if args.leaderboard_id:  # way to run only one leaderboard
-        query += f" #cleaderboard_id:{args.leaderboard_id}"
+        query += f" #leaderboard_id:{args.leaderboard_id}"
-
+    try:
-    leaderboards = bc.search(
+        leaderboards = bc.search(
-        token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+            token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
-        journal_id=MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
+            journal_id=MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
-        query=query,
+            query=query,
-        limit=100,
+            limit=100,
-        timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+            timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
-    )
+        )
+    except Exception as e:
+        logger.error(f"Could not get leaderboards from journal: {e}")
+        return
 
     if len(leaderboards.results) == 0:
-        raise ValueError("No leaderboard found")
+        logger.error("No leaderboard found")
+        return
 
     logger.info(f"Found {len(leaderboards.results)} leaderboards")
 
@@ -61,7 +71,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None:
             leaderboard_data = json.loads(leaderboard.content)
         except json.JSONDecodeError:
             logger.error(
-                f"Could not parse leaderboard content: {[tag for tag in leaderboard.tags if tag.startswith('leaderboard_id')]}"
+                f"Could not parse leaderboard content: {[tag for tag in leaderboard.tags if tag.startswith('leaderboard_id')]} in entry {leaderboard.entry_url.split('/')[-1]}"
             )
             continue
 
@@ -76,36 +86,48 @@ def handle_leaderboards(args: argparse.Namespace) -> None:
         else:
             params = leaderboard_data["params"]
 
-        ### execute query
+        blockchain = leaderboard_data.get("blockchain", None)
 
-        query_results = get_results_for_moonstream_query(
+        ### execute query
-            args.query_api_access_token,
+        try:
-            query_name,
+            query_results = get_results_for_moonstream_query(
-            params,
+                args.query_api_access_token,
-            args.query_api,
+                query_name,
-            args.max_retries,
+                params,
-            args.interval,
+                blockchain,
-        )
+                MOONSTREAM_API_URL,
+                args.max_retries,
+                args.interval,
+            )
+        except Exception as e:
+            logger.error(f"Could not get results for query {query_name}: error: {e}")
+            continue
 
         ### push results to leaderboard API
 
         if query_results is None:
-            logger.error(f"Could not get results for query {query_name}")
+            logger.error(f"Could not get results for query {query_name} in time")
             continue
 
-        leaderboard_push_api_url = f"{args.engine_api}/leaderboard/{leaderboard_id}/scores?normalize_addresses={leaderboard_data['normalize_addresses']}"
+        leaderboard_push_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/scores?normalize_addresses={leaderboard_data['normalize_addresses']}"
 
         leaderboard_api_headers = {
             "Authorization": f"Bearer {args.query_api_access_token}",
             "Content-Type": "application/json",
         }
 
-        leaderboard_api_response = requests.put(
+        try:
-            leaderboard_push_api_url,
+            leaderboard_api_response = requests.put(
-            json=query_results["data"],
+                leaderboard_push_api_url,
-            headers=leaderboard_api_headers,
+                json=query_results["data"],
-            timeout=10,
+                headers=leaderboard_api_headers,
-        )
+                timeout=10,
+            )
+        except Exception as e:
+            logger.error(
+                f"Could not push results to leaderboard API: {e} for leaderboard {leaderboard_id}"
+            )
+            continue
 
         try:
             leaderboard_api_response.raise_for_status()
@@ -118,7 +140,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None:
         ### get leaderboard from leaderboard API
 
         leaderboard_api_info_url = (
-            f"{args.engine_api}/leaderboard/info?leaderboard_id={leaderboard_id}"
+            f"{MOONSTREAM_ENGINE_URL}/leaderboard/info?leaderboard_id={leaderboard_id}"
         )
 
         leaderboard_api_response = requests.get(
@@ -136,16 +158,16 @@ def handle_leaderboards(args: argparse.Namespace) -> None:
         info = leaderboard_api_response.json()
 
         logger.info(
-            f"Successfully pushed results to leaderboard {info['id']}: {info['title']}"
+            f"Successfully pushed results to leaderboard {info['id']}:{blue_c} {info['title']} {end_c}"
         )
         logger.info(
-            f"can be check on:{args.engine_api}/leaderboard/?leaderboard_id={leaderboard_id}"
+            f"can be check on:{green_c} {MOONSTREAM_ENGINE_URL}/leaderboard/?leaderboard_id={leaderboard_id} {end_c}"
         )
 
 
 def main():
     """
-    Generates an argument parser for the "autocorns judge" command.
+    CLI for generating leaderboards from Moonstream Query API
     """
 
     parser = argparse.ArgumentParser(description="The Judge: Generate leaderboards")
@@ -155,16 +177,6 @@ def main():
     leaderboard_generator_parser = subparsers.add_parser(
         "leaderboards-generate", description="Generate Leaderboard"
     )
-    leaderboard_generator_parser.add_argument(
-        "--query-api",
-        default="https://api.moonstream.to",
-        help="Moonstream API URL.",
-    )
-    leaderboard_generator_parser.add_argument(
-        "--engine-api",
-        default="https://engineapi.moonstream.to",
-        help="Moonstream Engine API URL.",
-    )
     leaderboard_generator_parser.add_argument(
         "--leaderboard-id",
         type=uuid.UUID,

crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py

@@ -8,6 +8,8 @@ from typing import Any, Dict, Optional
 
 import requests  # type: ignore
 
+from ..settings import MOONSTREAM_API_URL
+
 
 logging.basicConfig()
 logger = logging.getLogger(__name__)
@@ -17,10 +19,32 @@ def get_results_for_moonstream_query(
     moonstream_access_token: str,
     query_name: str,
     params: Dict[str, Any],
-    api_url: str = "https://api.moonstream.to",
+    blockchain: Optional[str] = None,
+    api_url: str = MOONSTREAM_API_URL,
     max_retries: int = 100,
     interval: float = 30.0,
 ) -> Optional[Dict[str, Any]]:
+    """
+
+    Run update of query data and avaiting update of query result on S3.
+    TODO: Move to moonstream-client.
+
+    :param moonstream_access_token: Moonstream access token.
+
+    :param query_name: Name of the query to run.
+
+    :param params: Parameters to pass to the query.
+
+    :param api_url: URL of the Moonstream API.
+
+    :param max_retries: Maximum number of times to retry getting results from the Moonstream Query API.
+
+    :param interval: Number of seconds to wait between attempts to get results from the Moonstream Query API.
+
+    :return: Results of the query.
+
+    """
+
     result: Optional[Dict[str, Any]] = None
 
     api_url = api_url.rstrip("/")
@@ -33,7 +57,10 @@ def get_results_for_moonstream_query(
     if_modified_since_datetime = datetime.datetime.utcnow()
     if_modified_since = if_modified_since_datetime.strftime("%a, %d %b %Y %H:%M:%S GMT")
 
-    request_body = {"params": params}
+    request_body: Dict[str, Any] = {"params": params}
+
+    if blockchain is not None:
+        request_body["blockchain"] = blockchain
 
     success = False
     attempts = 0

crawlers/mooncrawl/mooncrawl/settings.py

@@ -23,6 +23,11 @@ if MOONSTREAM_ENTITY_URL == "":
 
 entity_client = Entity(MOONSTREAM_ENTITY_URL)
 
+MOONSTREAM_API_URL = os.environ.get("MOONSTREAM_API_URL", "https://api.moonstream.to")
+MOONSTREAM_ENGINE_URL = os.environ.get(
+    "MOONSTREAM_ENGINE_URL", "https://engineapi.moonstream.to"
+)
+
 
 BUGOUT_REQUEST_TIMEOUT_SECONDS_RAW = os.environ.get(
     "MOONSTREAM_BUGOUT_TIMEOUT_SECONDS", 30

crawlers/mooncrawl/sample.env

@@ -7,7 +7,11 @@ export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout_Humbug_token_for_crash_reports>"
 # Entity environment variables
 export MOONSTREAM_ENTITY_URL="https://api.moonstream.to/entity"
 
+# Engine environment variables
+export MOONSTREAM_ENGINE_URL="https://engineapi.moonstream.to"
+
 # Moonstream environment variables
+export MOONSTREAM_API_URL="https://api.moonstream.to"
 export MOONSTREAM_BUGOUT_TIMEOUT_SECONDS=30
 export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to"
 export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"