From 51359548e46a2ec3dbc268d39179a7008c281886 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 6 Nov 2023 20:10:45 +0200 Subject: [PATCH] Add changes. --- crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py | 7 +++++++ .../mooncrawl/mooncrawl/leaderboards_generator/utils.py | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py index 2908305b..39b8f0f7 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -97,6 +97,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: MOONSTREAM_API_URL, args.max_retries, args.interval, + args.query_api_retries, ) except Exception as e: logger.error(f"Could not get results for query {query_name}: error: {e}") @@ -188,6 +189,12 @@ def main(): default=12, help="Number of times to retry requests for Moonstream Query results", ) + leaderboard_generator_parser.add_argument( + "--query-api-retries", + type=int, + default=3, + help="Number of times to retry updating Moonstream Query data", + ) leaderboard_generator_parser.add_argument( "--interval", type=float, diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py index fa235cca..fc4d1e01 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -23,6 +23,7 @@ def get_results_for_moonstream_query( api_url: str = MOONSTREAM_API_URL, max_retries: int = 100, interval: float = 30.0, + query_api_retries: int = 3, ) -> Optional[Dict[str, Any]]: """ @@ -65,11 +66,11 @@ def get_results_for_moonstream_query( success = False attempts = 0 - while not success and attempts < max_retries: - attempts += 1 + while not success and attempts < query_api_retries: response = requests.post( request_url, json=request_body, headers=headers, timeout=10 ) + attempts += 1 response.raise_for_status() response_body = response.json() data_url = response_body["url"]