Merge pull request #965 from moonstream-to/add-query-repeat-parameter

Add strict query api request repeat for leaderboard.

crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py

@@ -97,6 +97,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None:
                 MOONSTREAM_API_URL,
                 args.max_retries,
                 args.interval,
+                args.query_api_retries,
             )
         except Exception as e:
             logger.error(f"Could not get results for query {query_name}: error: {e}")
@@ -188,6 +189,12 @@ def main():
         default=12,
         help="Number of times to retry requests for Moonstream Query results",
     )
+    leaderboard_generator_parser.add_argument(
+        "--query-api-retries",
+        type=int,
+        default=3,
+        help="Number of times to retry updating Moonstream Query data",
+    )
     leaderboard_generator_parser.add_argument(
         "--interval",
         type=float,

crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py

@@ -23,6 +23,7 @@ def get_results_for_moonstream_query(
     api_url: str = MOONSTREAM_API_URL,
     max_retries: int = 100,
     interval: float = 30.0,
+    query_api_retries: int = 3,
 ) -> Optional[Dict[str, Any]]:
     """
 
@@ -65,11 +66,11 @@ def get_results_for_moonstream_query(
     success = False
     attempts = 0
 
-    while not success and attempts < max_retries:
-        attempts += 1
+    while not success and attempts < query_api_retries:
         response = requests.post(
             request_url, json=request_body, headers=headers, timeout=10
         )
+        attempts += 1
         response.raise_for_status()
         response_body = response.json()
         data_url = response_body["url"]