diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py index 2908305b..39b8f0f7 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -97,6 +97,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: MOONSTREAM_API_URL, args.max_retries, args.interval, + args.query_api_retries, ) except Exception as e: logger.error(f"Could not get results for query {query_name}: error: {e}") @@ -188,6 +189,12 @@ def main(): default=12, help="Number of times to retry requests for Moonstream Query results", ) + leaderboard_generator_parser.add_argument( + "--query-api-retries", + type=int, + default=3, + help="Number of times to retry updating Moonstream Query data", + ) leaderboard_generator_parser.add_argument( "--interval", type=float, diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py index fa235cca..fc4d1e01 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -23,6 +23,7 @@ def get_results_for_moonstream_query( api_url: str = MOONSTREAM_API_URL, max_retries: int = 100, interval: float = 30.0, + query_api_retries: int = 3, ) -> Optional[Dict[str, Any]]: """ @@ -65,11 +66,11 @@ def get_results_for_moonstream_query( success = False attempts = 0 - while not success and attempts < max_retries: - attempts += 1 + while not success and attempts < query_api_retries: response = requests.post( request_url, json=request_body, headers=headers, timeout=10 ) + attempts += 1 response.raise_for_status() response_body = response.json() data_url = response_body["url"]