From cb1137fd3992deca13cb5c5638c94fdb81605c74 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 26 Jul 2023 16:21:56 +0300 Subject: [PATCH 1/7] Add init version. --- .../leaderboard_generator/__init__.py | 0 .../mooncrawl/leaderboard_generator/cli.py | 196 ++++++++++++++++++ .../mooncrawl/leaderboard_generator/utils.py | 75 +++++++ crawlers/mooncrawl/mooncrawl/settings.py | 11 + 4 files changed, 282 insertions(+) create mode 100644 crawlers/mooncrawl/mooncrawl/leaderboard_generator/__init__.py create mode 100644 crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py create mode 100644 crawlers/mooncrawl/mooncrawl/leaderboard_generator/utils.py diff --git a/crawlers/mooncrawl/mooncrawl/leaderboard_generator/__init__.py b/crawlers/mooncrawl/mooncrawl/leaderboard_generator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py new file mode 100644 index 00000000..7f2e57ec --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py @@ -0,0 +1,196 @@ +import argparse +import json +import logging +import os +from typing import Any, Dict +import uuid + +import requests # type: ignore + + +from .utils import get_results_for_moonstream_query +from ..settings import ( + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, + BUGOUT_REQUEST_TIMEOUT_SECONDS, +) + +from ..settings import bugout_client as bc + + +logging.basicConfig() +logger = logging.getLogger(__name__) + + +def handle_leaderboards(args: argparse.Namespace) -> None: + """ + Run the leaderboard generator. + + Get query from journal and push results to leaderboard API. + """ + + ### get leaderboard journal + + query = "#leaderboard" + + if args.leaderboard_id: + query += f" leaderboard_id:{args.leaderboard_id}" + + leaderboards = bc.search( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, + query=query, + limit=1, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + + if len(leaderboards.results) == 0: + raise ValueError("No leaderboard found") + + for leaderboard in leaderboards.results: + if leaderboard.content is None: + continue + + try: + leaderboard_data = json.loads(leaderboard.content) + except json.JSONDecodeError: + logger.error( + f"Could not parse leaderboard content: {[tag for tag in leaderboard.tags if tag.startswith('leaderboard_id')]}" + ) + continue + + ### get results from query API + + leaderboard_id = leaderboard_data["leaderboard_id"] + + query_name = leaderboard_data["query_name"] + + if args.params: + params = json.loads(args.params) + else: + params = leaderboard_data["params"] + + ### execute query + + query_results = get_results_for_moonstream_query( + args.query_api_access_token, + args.query_name, + params, + args.query_api, + args.max_retries, + args.interval, + ) + + ### push results to leaderboard API + + if query_results is None: + logger.error(f"Could not get results for query {query_name}") + continue + + leaderboard_push_api_url = f"{args.engine_api}/leaderboards/{leaderboard_id}" + + leaderboard_api_headers = { + "Authorization": f"Bearer {args.query_api_access_token}", + "Content-Type": "application/json", + } + + leaderboard_api_response = requests.put( + leaderboard_push_api_url, + json=query_results, + headers=leaderboard_api_headers, + timeout=10, + ) + + try: + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" + ) + continue + + ### get leaderboard from leaderboard API + + leaderboard_api_info_url = ( + f"{args.engine_api}/leaderboards/info?leaderboard_id={leaderboard_id}" + ) + + leaderboard_api_response = requests.get( + leaderboard_api_info_url, headers=leaderboard_api_headers, timeout=10 + ) + + try: + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not get leaderboard info from leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" + ) + continue + + info = leaderboard_api_response.json() + + logger.info( + f"Successfully pushed results to leaderboard {info['id']}: {info['name']}" + ) + + +def generate_cli() -> argparse.ArgumentParser: + """ + Generates an argument parser for the "autocorns judge" command. + """ + parser = argparse.ArgumentParser(description="The Judge: Generate leaderboards") + parser.set_defaults(func=lambda _: parser.print_help()) + subparsers = parser.add_subparsers() + + shadowcorns_throwing_shade_parser = subparsers.add_parser( + "throwing-shade", description="Shadowcorns: Throwing Shade Leaderboard" + ) + shadowcorns_throwing_shade_parser.add_argument( + "--query-api", + default="https://api.moonstream.to", + help="Moonstream API URL. Access token expected to be set as MOONSTREAM_ACCESS_TOKEN environment variable.", + ) + shadowcorns_throwing_shade_parser.add_argument( + "--engine-api", + default="https://engineapi.moonstream.to", + help="Moonstream Engine API URL. Access token expected to be set as MOONSTREAM_ACCESS_TOKEN environment variable.", + ) + shadowcorns_throwing_shade_parser.add_argument( + "--leaderboard-id", + type=uuid.UUID, + required=True, + help="Leaderboard ID on Engine API", + ) + shadowcorns_throwing_shade_parser.add_argument( + "--max-retries", + type=int, + default=100, + help="Number of times to retry requests for Moonstream Query results", + ) + shadowcorns_throwing_shade_parser.add_argument( + "--interval", + type=float, + default=30.0, + help="Number of seconds to wait between attempts to get results from Moonstream Query API", + ) + shadowcorns_throwing_shade_parser.add_argument( + "--params", + type=json.loads, + help="Parameters to pass to Moonstream Query API", + ) + shadowcorns_throwing_shade_parser.add_argument( + "--query-api-access-token", + type=str, + required=True, + help="Moonstream Access Token to use for Moonstream Query API requests", + ) + + shadowcorns_throwing_shade_parser.set_defaults(func=handle_leaderboards) + + return parser + + +if __name__ == "__main__": + parser = generate_cli() + args = parser.parse_args() + args.func(args) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboard_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboard_generator/utils.py new file mode 100644 index 00000000..8ec3013b --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/leaderboard_generator/utils.py @@ -0,0 +1,75 @@ +import datetime +import json +import logging +import os +import time +from typing import Any, Dict, Optional + + +import requests # type: ignore + + +logging.basicConfig() +logger = logging.getLogger(__name__) + + +def get_results_for_moonstream_query( + moonstream_access_token: str, + query_name: str, + params: Dict[str, Any], + api_url: str = "https://api.moonstream.to", + max_retries: int = 100, + interval: float = 30.0, +) -> Optional[Dict[str, Any]]: + result: Optional[Dict[str, Any]] = None + + api_url = api_url.rstrip("/") + request_url = f"{api_url}/queries/{query_name}/update_data" + headers = { + "Authorization": f"Bearer {moonstream_access_token}", + "Content-Type": "application/json", + } + # Assume our clock is not drifting too much from AWS clocks. + if_modified_since_datetime = datetime.datetime.utcnow() + if_modified_since = if_modified_since_datetime.strftime("%a, %d %b %Y %H:%M:%S GMT") + + request_body = {"params": params} + + success = False + attempts = 0 + + while not success and attempts < max_retries: + attempts += 1 + response = requests.post( + request_url, json=request_body, headers=headers, timeout=10 + ) + response.raise_for_status() + response_body = response.json() + data_url = response_body["url"] + + keep_going = True + num_retries = 0 + + logging.debug(f"If-Modified-Since: {if_modified_since}") + while keep_going: + time.sleep(interval) + num_retries += 1 + try: + data_response = requests.get( + data_url, + headers={"If-Modified-Since": if_modified_since}, + timeout=10, + ) + except: + logger.error(f"Failed to get data from {data_url}") + continue + logger.debug(f"Status code: {data_response.status_code}") + logger.debug(f"Last-Modified: {data_response.headers['Last-Modified']}") + if data_response.status_code == 200: + result = data_response.json() + keep_going = False + success = True + if keep_going and max_retries > 0: + keep_going = num_retries <= max_retries + + return result diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index b0d36b89..f6d43f39 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -310,3 +310,14 @@ HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES = { "historical_crawl_status": "historical_crawl_status", "progress_status": "progress", } + + +# Leaderboard generator + +MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID = os.environ.get( + "MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID", "" +) +if MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID == "": + raise ValueError( + "MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID environment variable must be set" + ) From 6c11fb3d0f0090d3cee4407f7f851f2fa9e5940c Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 26 Jul 2023 20:20:17 +0300 Subject: [PATCH 2/7] Add working leaderboard update. --- .../mooncrawl/leaderboard_generator/cli.py | 29 ++++++++++--------- crawlers/mooncrawl/setup.py | 1 + 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py index 7f2e57ec..9aaad6d2 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py @@ -34,7 +34,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: query = "#leaderboard" if args.leaderboard_id: - query += f" leaderboard_id:{args.leaderboard_id}" + query += f" #cleaderboard_id:{args.leaderboard_id}" leaderboards = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -74,7 +74,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: query_results = get_results_for_moonstream_query( args.query_api_access_token, - args.query_name, + query_name, params, args.query_api, args.max_retries, @@ -87,7 +87,9 @@ def handle_leaderboards(args: argparse.Namespace) -> None: logger.error(f"Could not get results for query {query_name}") continue - leaderboard_push_api_url = f"{args.engine_api}/leaderboards/{leaderboard_id}" + leaderboard_push_api_url = ( + f"{args.engine_api}/leaderboard/{leaderboard_id}/scores" + ) leaderboard_api_headers = { "Authorization": f"Bearer {args.query_api_access_token}", @@ -96,7 +98,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: leaderboard_api_response = requests.put( leaderboard_push_api_url, - json=query_results, + json=query_results["data"], headers=leaderboard_api_headers, timeout=10, ) @@ -112,7 +114,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: ### get leaderboard from leaderboard API leaderboard_api_info_url = ( - f"{args.engine_api}/leaderboards/info?leaderboard_id={leaderboard_id}" + f"{args.engine_api}/leaderboard/info?leaderboard_id={leaderboard_id}" ) leaderboard_api_response = requests.get( @@ -130,20 +132,21 @@ def handle_leaderboards(args: argparse.Namespace) -> None: info = leaderboard_api_response.json() logger.info( - f"Successfully pushed results to leaderboard {info['id']}: {info['name']}" + f"Successfully pushed results to leaderboard {info['id']}: {info['title']}" ) -def generate_cli() -> argparse.ArgumentParser: +def main(): """ Generates an argument parser for the "autocorns judge" command. """ + parser = argparse.ArgumentParser(description="The Judge: Generate leaderboards") parser.set_defaults(func=lambda _: parser.print_help()) subparsers = parser.add_subparsers() shadowcorns_throwing_shade_parser = subparsers.add_parser( - "throwing-shade", description="Shadowcorns: Throwing Shade Leaderboard" + "leaderboards-generate", description="Generate Leaderboard" ) shadowcorns_throwing_shade_parser.add_argument( "--query-api", @@ -158,7 +161,7 @@ def generate_cli() -> argparse.ArgumentParser: shadowcorns_throwing_shade_parser.add_argument( "--leaderboard-id", type=uuid.UUID, - required=True, + required=False, help="Leaderboard ID on Engine API", ) shadowcorns_throwing_shade_parser.add_argument( @@ -176,6 +179,7 @@ def generate_cli() -> argparse.ArgumentParser: shadowcorns_throwing_shade_parser.add_argument( "--params", type=json.loads, + required=False, help="Parameters to pass to Moonstream Query API", ) shadowcorns_throwing_shade_parser.add_argument( @@ -187,10 +191,9 @@ def generate_cli() -> argparse.ArgumentParser: shadowcorns_throwing_shade_parser.set_defaults(func=handle_leaderboards) - return parser + args = parser.parse_args() + args.func(args) if __name__ == "__main__": - parser = generate_cli() - args = parser.parse_args() - args.func(args) + main() diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index fa9586d5..b7e42dfc 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -67,6 +67,7 @@ setup( "state-crawler=mooncrawl.state_crawler.cli:main", "metadata-crawler=mooncrawl.metadata_crawler.cli:main", "custom-crawler=mooncrawl.reports_crawler.cli:main", + "leaderboard-generator=mooncrawl.leaderboard_generator.cli:main", ] }, ) From 4d1d242c813eda2ba1886d5e0ce3ec28d535c137 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 26 Jul 2023 23:51:15 +0300 Subject: [PATCH 3/7] Add loggers and fixes. --- crawlers/deploy/deploy.bash | 14 ++ crawlers/deploy/leaderboards-worker.service | 11 + crawlers/deploy/leaderboards-worker.timer | 9 + .../leaderboard_generator/__init__.py | 0 .../mooncrawl/leaderboard_generator/cli.py | 199 ------------------ .../mooncrawl/leaderboard_generator/utils.py | 75 ------- crawlers/mooncrawl/mooncrawl/version.py | 2 +- crawlers/mooncrawl/setup.py | 2 +- 8 files changed, 36 insertions(+), 276 deletions(-) create mode 100644 crawlers/deploy/leaderboards-worker.service create mode 100644 crawlers/deploy/leaderboards-worker.timer delete mode 100644 crawlers/mooncrawl/mooncrawl/leaderboard_generator/__init__.py delete mode 100644 crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py delete mode 100644 crawlers/mooncrawl/mooncrawl/leaderboard_generator/utils.py diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 6df24df1..c0f3545f 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -26,6 +26,9 @@ SCRIPT_DIR="$(realpath $(dirname $0))" # Service files MOONCRAWL_SERVICE_FILE="mooncrawl.service" +LEADERBOARDS_WORKER_SERVICE_FILE="leaderboards-worker.service" +LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer" + # Ethereum service files ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service" @@ -553,3 +556,14 @@ cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/ cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" + + + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Leaderboards worker service and timer with: ${LEADERBOARDS_WORKER_SERVICE_FILE}, ${LEADERBOARDS_WORKER_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" +cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}" \ No newline at end of file diff --git a/crawlers/deploy/leaderboards-worker.service b/crawlers/deploy/leaderboards-worker.service new file mode 100644 index 00000000..0f70e80e --- /dev/null +++ b/crawlers/deploy/leaderboards-worker.service @@ -0,0 +1,11 @@ +[Unit] +Description=Runs leaderboards generator worker +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.leaderboards_generator.cli leaderboards-generate --query-api-access-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" +CPUWeight=60 +SyslogIdentifier=leaderboards-generator \ No newline at end of file diff --git a/crawlers/deploy/leaderboards-worker.timer b/crawlers/deploy/leaderboards-worker.timer new file mode 100644 index 00000000..108f5175 --- /dev/null +++ b/crawlers/deploy/leaderboards-worker.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs leaderboard update script every 10 minutes + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target diff --git a/crawlers/mooncrawl/mooncrawl/leaderboard_generator/__init__.py b/crawlers/mooncrawl/mooncrawl/leaderboard_generator/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py deleted file mode 100644 index 9aaad6d2..00000000 --- a/crawlers/mooncrawl/mooncrawl/leaderboard_generator/cli.py +++ /dev/null @@ -1,199 +0,0 @@ -import argparse -import json -import logging -import os -from typing import Any, Dict -import uuid - -import requests # type: ignore - - -from .utils import get_results_for_moonstream_query -from ..settings import ( - MOONSTREAM_ADMIN_ACCESS_TOKEN, - MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, - BUGOUT_REQUEST_TIMEOUT_SECONDS, -) - -from ..settings import bugout_client as bc - - -logging.basicConfig() -logger = logging.getLogger(__name__) - - -def handle_leaderboards(args: argparse.Namespace) -> None: - """ - Run the leaderboard generator. - - Get query from journal and push results to leaderboard API. - """ - - ### get leaderboard journal - - query = "#leaderboard" - - if args.leaderboard_id: - query += f" #cleaderboard_id:{args.leaderboard_id}" - - leaderboards = bc.search( - token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - journal_id=MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, - query=query, - limit=1, - timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, - ) - - if len(leaderboards.results) == 0: - raise ValueError("No leaderboard found") - - for leaderboard in leaderboards.results: - if leaderboard.content is None: - continue - - try: - leaderboard_data = json.loads(leaderboard.content) - except json.JSONDecodeError: - logger.error( - f"Could not parse leaderboard content: {[tag for tag in leaderboard.tags if tag.startswith('leaderboard_id')]}" - ) - continue - - ### get results from query API - - leaderboard_id = leaderboard_data["leaderboard_id"] - - query_name = leaderboard_data["query_name"] - - if args.params: - params = json.loads(args.params) - else: - params = leaderboard_data["params"] - - ### execute query - - query_results = get_results_for_moonstream_query( - args.query_api_access_token, - query_name, - params, - args.query_api, - args.max_retries, - args.interval, - ) - - ### push results to leaderboard API - - if query_results is None: - logger.error(f"Could not get results for query {query_name}") - continue - - leaderboard_push_api_url = ( - f"{args.engine_api}/leaderboard/{leaderboard_id}/scores" - ) - - leaderboard_api_headers = { - "Authorization": f"Bearer {args.query_api_access_token}", - "Content-Type": "application/json", - } - - leaderboard_api_response = requests.put( - leaderboard_push_api_url, - json=query_results["data"], - headers=leaderboard_api_headers, - timeout=10, - ) - - try: - leaderboard_api_response.raise_for_status() - except requests.exceptions.HTTPError as http_error: - logger.error( - f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" - ) - continue - - ### get leaderboard from leaderboard API - - leaderboard_api_info_url = ( - f"{args.engine_api}/leaderboard/info?leaderboard_id={leaderboard_id}" - ) - - leaderboard_api_response = requests.get( - leaderboard_api_info_url, headers=leaderboard_api_headers, timeout=10 - ) - - try: - leaderboard_api_response.raise_for_status() - except requests.exceptions.HTTPError as http_error: - logger.error( - f"Could not get leaderboard info from leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" - ) - continue - - info = leaderboard_api_response.json() - - logger.info( - f"Successfully pushed results to leaderboard {info['id']}: {info['title']}" - ) - - -def main(): - """ - Generates an argument parser for the "autocorns judge" command. - """ - - parser = argparse.ArgumentParser(description="The Judge: Generate leaderboards") - parser.set_defaults(func=lambda _: parser.print_help()) - subparsers = parser.add_subparsers() - - shadowcorns_throwing_shade_parser = subparsers.add_parser( - "leaderboards-generate", description="Generate Leaderboard" - ) - shadowcorns_throwing_shade_parser.add_argument( - "--query-api", - default="https://api.moonstream.to", - help="Moonstream API URL. Access token expected to be set as MOONSTREAM_ACCESS_TOKEN environment variable.", - ) - shadowcorns_throwing_shade_parser.add_argument( - "--engine-api", - default="https://engineapi.moonstream.to", - help="Moonstream Engine API URL. Access token expected to be set as MOONSTREAM_ACCESS_TOKEN environment variable.", - ) - shadowcorns_throwing_shade_parser.add_argument( - "--leaderboard-id", - type=uuid.UUID, - required=False, - help="Leaderboard ID on Engine API", - ) - shadowcorns_throwing_shade_parser.add_argument( - "--max-retries", - type=int, - default=100, - help="Number of times to retry requests for Moonstream Query results", - ) - shadowcorns_throwing_shade_parser.add_argument( - "--interval", - type=float, - default=30.0, - help="Number of seconds to wait between attempts to get results from Moonstream Query API", - ) - shadowcorns_throwing_shade_parser.add_argument( - "--params", - type=json.loads, - required=False, - help="Parameters to pass to Moonstream Query API", - ) - shadowcorns_throwing_shade_parser.add_argument( - "--query-api-access-token", - type=str, - required=True, - help="Moonstream Access Token to use for Moonstream Query API requests", - ) - - shadowcorns_throwing_shade_parser.set_defaults(func=handle_leaderboards) - - args = parser.parse_args() - args.func(args) - - -if __name__ == "__main__": - main() diff --git a/crawlers/mooncrawl/mooncrawl/leaderboard_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboard_generator/utils.py deleted file mode 100644 index 8ec3013b..00000000 --- a/crawlers/mooncrawl/mooncrawl/leaderboard_generator/utils.py +++ /dev/null @@ -1,75 +0,0 @@ -import datetime -import json -import logging -import os -import time -from typing import Any, Dict, Optional - - -import requests # type: ignore - - -logging.basicConfig() -logger = logging.getLogger(__name__) - - -def get_results_for_moonstream_query( - moonstream_access_token: str, - query_name: str, - params: Dict[str, Any], - api_url: str = "https://api.moonstream.to", - max_retries: int = 100, - interval: float = 30.0, -) -> Optional[Dict[str, Any]]: - result: Optional[Dict[str, Any]] = None - - api_url = api_url.rstrip("/") - request_url = f"{api_url}/queries/{query_name}/update_data" - headers = { - "Authorization": f"Bearer {moonstream_access_token}", - "Content-Type": "application/json", - } - # Assume our clock is not drifting too much from AWS clocks. - if_modified_since_datetime = datetime.datetime.utcnow() - if_modified_since = if_modified_since_datetime.strftime("%a, %d %b %Y %H:%M:%S GMT") - - request_body = {"params": params} - - success = False - attempts = 0 - - while not success and attempts < max_retries: - attempts += 1 - response = requests.post( - request_url, json=request_body, headers=headers, timeout=10 - ) - response.raise_for_status() - response_body = response.json() - data_url = response_body["url"] - - keep_going = True - num_retries = 0 - - logging.debug(f"If-Modified-Since: {if_modified_since}") - while keep_going: - time.sleep(interval) - num_retries += 1 - try: - data_response = requests.get( - data_url, - headers={"If-Modified-Since": if_modified_since}, - timeout=10, - ) - except: - logger.error(f"Failed to get data from {data_url}") - continue - logger.debug(f"Status code: {data_response.status_code}") - logger.debug(f"Last-Modified: {data_response.headers['Last-Modified']}") - if data_response.status_code == 200: - result = data_response.json() - keep_going = False - success = True - if keep_going and max_retries > 0: - keep_going = num_retries <= max_retries - - return result diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index f04f8d7c..2d9a5a21 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.3.2" +MOONCRAWL_VERSION = "0.3.3" diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index b7e42dfc..95751fe2 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -67,7 +67,7 @@ setup( "state-crawler=mooncrawl.state_crawler.cli:main", "metadata-crawler=mooncrawl.metadata_crawler.cli:main", "custom-crawler=mooncrawl.reports_crawler.cli:main", - "leaderboard-generator=mooncrawl.leaderboard_generator.cli:main", + "leaderboards-generator=mooncrawl.leaderboards_generator.cli:main", ] }, ) From 3215e5d7190fdeead72dbb4b2415ed8e9ea72c75 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 26 Jul 2023 23:52:35 +0300 Subject: [PATCH 4/7] Add leaderboards dir. --- .../leaderboards_generator/__init__.py | 0 .../mooncrawl/leaderboards_generator/cli.py | 206 ++++++++++++++++++ .../mooncrawl/leaderboards_generator/utils.py | 75 +++++++ 3 files changed, 281 insertions(+) create mode 100644 crawlers/mooncrawl/mooncrawl/leaderboards_generator/__init__.py create mode 100644 crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py create mode 100644 crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/__init__.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py new file mode 100644 index 00000000..3011b1e4 --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -0,0 +1,206 @@ +import argparse +import json +import logging +import os +from typing import Any, Dict +import uuid + +import requests # type: ignore + + +from .utils import get_results_for_moonstream_query +from ..settings import ( + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, + BUGOUT_REQUEST_TIMEOUT_SECONDS, +) + +from ..settings import bugout_client as bc + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def handle_leaderboards(args: argparse.Namespace) -> None: + """ + Run the leaderboard generator. + + Get query from journal and push results to leaderboard API. + """ + + ### get leaderboard journal + + query = "#leaderboard" + + if args.leaderboard_id: + query += f" #cleaderboard_id:{args.leaderboard_id}" + + leaderboards = bc.search( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, + query=query, + limit=100, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + + if len(leaderboards.results) == 0: + raise ValueError("No leaderboard found") + + logger.info(f"Found {len(leaderboards.results)} leaderboards") + + for leaderboard in leaderboards.results: + logger.info( + f"Processing leaderboard: {leaderboard.title} with id: {[tag for tag in leaderboard.tags if tag.startswith('leaderboard_id')]}" + ) + + if leaderboard.content is None: + continue + + try: + leaderboard_data = json.loads(leaderboard.content) + except json.JSONDecodeError: + logger.error( + f"Could not parse leaderboard content: {[tag for tag in leaderboard.tags if tag.startswith('leaderboard_id')]}" + ) + continue + + ### get results from query API + + leaderboard_id = leaderboard_data["leaderboard_id"] + + query_name = leaderboard_data["query_name"] + + if args.params: + params = json.loads(args.params) + else: + params = leaderboard_data["params"] + + ### execute query + + query_results = get_results_for_moonstream_query( + args.query_api_access_token, + query_name, + params, + args.query_api, + args.max_retries, + args.interval, + ) + + ### push results to leaderboard API + + if query_results is None: + logger.error(f"Could not get results for query {query_name}") + continue + + leaderboard_push_api_url = f"{args.engine_api}/leaderboard/{leaderboard_id}/scores?normalize_addresses={leaderboard_data['normalize_addresses']}" + + leaderboard_api_headers = { + "Authorization": f"Bearer {args.query_api_access_token}", + "Content-Type": "application/json", + } + + leaderboard_api_response = requests.put( + leaderboard_push_api_url, + json=query_results["data"], + headers=leaderboard_api_headers, + timeout=10, + ) + + try: + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" + ) + continue + + ### get leaderboard from leaderboard API + + leaderboard_api_info_url = ( + f"{args.engine_api}/leaderboard/info?leaderboard_id={leaderboard_id}" + ) + + leaderboard_api_response = requests.get( + leaderboard_api_info_url, headers=leaderboard_api_headers, timeout=10 + ) + + try: + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not get leaderboard info from leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" + ) + continue + + info = leaderboard_api_response.json() + + logger.info( + f"Successfully pushed results to leaderboard {info['id']}: {info['title']}" + ) + logger.info( + f"can be check on:{args.engine_api}/leaderboard/?leaderboard_id={leaderboard_id}" + ) + + +def main(): + """ + Generates an argument parser for the "autocorns judge" command. + """ + + parser = argparse.ArgumentParser(description="The Judge: Generate leaderboards") + parser.set_defaults(func=lambda _: parser.print_help()) + subparsers = parser.add_subparsers() + + leaderboard_generator_parser = subparsers.add_parser( + "leaderboards-generate", description="Generate Leaderboard" + ) + leaderboard_generator_parser.add_argument( + "--query-api", + default="https://api.moonstream.to", + help="Moonstream API URL.", + ) + leaderboard_generator_parser.add_argument( + "--engine-api", + default="https://engineapi.moonstream.to", + help="Moonstream Engine API URL.", + ) + leaderboard_generator_parser.add_argument( + "--leaderboard-id", + type=uuid.UUID, + required=False, + help="Leaderboard ID on Engine API", + ) + leaderboard_generator_parser.add_argument( + "--max-retries", + type=int, + default=100, + help="Number of times to retry requests for Moonstream Query results", + ) + leaderboard_generator_parser.add_argument( + "--interval", + type=float, + default=30.0, + help="Number of seconds to wait between attempts to get results from Moonstream Query API", + ) + leaderboard_generator_parser.add_argument( + "--params", + type=json.loads, + required=False, + help="Parameters to pass to Moonstream Query API. Use together with --leaderboard-id", + ) + leaderboard_generator_parser.add_argument( + "--query-api-access-token", + type=str, + required=True, + help="Moonstream Access Token to use for Moonstream Query API requests", + ) + + leaderboard_generator_parser.set_defaults(func=handle_leaderboards) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py new file mode 100644 index 00000000..8ec3013b --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -0,0 +1,75 @@ +import datetime +import json +import logging +import os +import time +from typing import Any, Dict, Optional + + +import requests # type: ignore + + +logging.basicConfig() +logger = logging.getLogger(__name__) + + +def get_results_for_moonstream_query( + moonstream_access_token: str, + query_name: str, + params: Dict[str, Any], + api_url: str = "https://api.moonstream.to", + max_retries: int = 100, + interval: float = 30.0, +) -> Optional[Dict[str, Any]]: + result: Optional[Dict[str, Any]] = None + + api_url = api_url.rstrip("/") + request_url = f"{api_url}/queries/{query_name}/update_data" + headers = { + "Authorization": f"Bearer {moonstream_access_token}", + "Content-Type": "application/json", + } + # Assume our clock is not drifting too much from AWS clocks. + if_modified_since_datetime = datetime.datetime.utcnow() + if_modified_since = if_modified_since_datetime.strftime("%a, %d %b %Y %H:%M:%S GMT") + + request_body = {"params": params} + + success = False + attempts = 0 + + while not success and attempts < max_retries: + attempts += 1 + response = requests.post( + request_url, json=request_body, headers=headers, timeout=10 + ) + response.raise_for_status() + response_body = response.json() + data_url = response_body["url"] + + keep_going = True + num_retries = 0 + + logging.debug(f"If-Modified-Since: {if_modified_since}") + while keep_going: + time.sleep(interval) + num_retries += 1 + try: + data_response = requests.get( + data_url, + headers={"If-Modified-Since": if_modified_since}, + timeout=10, + ) + except: + logger.error(f"Failed to get data from {data_url}") + continue + logger.debug(f"Status code: {data_response.status_code}") + logger.debug(f"Last-Modified: {data_response.headers['Last-Modified']}") + if data_response.status_code == 200: + result = data_response.json() + keep_going = False + success = True + if keep_going and max_retries > 0: + keep_going = num_retries <= max_retries + + return result From bf7f6497c549e5e0482ee32a3343650e3142c8d3 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 27 Jul 2023 06:59:01 +0300 Subject: [PATCH 5/7] Add changes. --- crawlers/deploy/leaderboards-worker.service | 2 +- .../mooncrawl/leaderboards_generator/cli.py | 98 +++++++++++-------- .../mooncrawl/leaderboards_generator/utils.py | 31 +++++- crawlers/mooncrawl/mooncrawl/settings.py | 5 + crawlers/mooncrawl/sample.env | 4 + 5 files changed, 94 insertions(+), 46 deletions(-) diff --git a/crawlers/deploy/leaderboards-worker.service b/crawlers/deploy/leaderboards-worker.service index 0f70e80e..2f7f6d06 100644 --- a/crawlers/deploy/leaderboards-worker.service +++ b/crawlers/deploy/leaderboards-worker.service @@ -8,4 +8,4 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.leaderboards_generator.cli leaderboards-generate --query-api-access-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" CPUWeight=60 -SyslogIdentifier=leaderboards-generator \ No newline at end of file +SyslogIdentifier=leaderboards-worker \ No newline at end of file diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py index 3011b1e4..f20fe185 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -13,6 +13,8 @@ from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, BUGOUT_REQUEST_TIMEOUT_SECONDS, + MOONSTREAM_API_URL, + MOONSTREAM_ENGINE_URL, ) from ..settings import bugout_client as bc @@ -21,6 +23,10 @@ from ..settings import bugout_client as bc logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +blue_c = "\033[94m" +green_c = "\033[92m" +end_c = "\033[0m" + def handle_leaderboards(args: argparse.Namespace) -> None: """ @@ -33,19 +39,23 @@ def handle_leaderboards(args: argparse.Namespace) -> None: query = "#leaderboard" - if args.leaderboard_id: - query += f" #cleaderboard_id:{args.leaderboard_id}" - - leaderboards = bc.search( - token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - journal_id=MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, - query=query, - limit=100, - timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, - ) + if args.leaderboard_id: # way to run only one leaderboard + query += f" #leaderboard_id:{args.leaderboard_id}" + try: + leaderboards = bc.search( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + journal_id=MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, + query=query, + limit=100, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + except Exception as e: + logger.error(f"Could not get leaderboards from journal: {e}") + return if len(leaderboards.results) == 0: - raise ValueError("No leaderboard found") + logger.error("No leaderboard found") + return logger.info(f"Found {len(leaderboards.results)} leaderboards") @@ -61,7 +71,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: leaderboard_data = json.loads(leaderboard.content) except json.JSONDecodeError: logger.error( - f"Could not parse leaderboard content: {[tag for tag in leaderboard.tags if tag.startswith('leaderboard_id')]}" + f"Could not parse leaderboard content: {[tag for tag in leaderboard.tags if tag.startswith('leaderboard_id')]} in entry {leaderboard.entry_url.split('/')[-1]}" ) continue @@ -76,36 +86,48 @@ def handle_leaderboards(args: argparse.Namespace) -> None: else: params = leaderboard_data["params"] - ### execute query + blockchain = leaderboard_data.get("blockchain", None) - query_results = get_results_for_moonstream_query( - args.query_api_access_token, - query_name, - params, - args.query_api, - args.max_retries, - args.interval, - ) + ### execute query + try: + query_results = get_results_for_moonstream_query( + args.query_api_access_token, + query_name, + params, + blockchain, + MOONSTREAM_API_URL, + args.max_retries, + args.interval, + ) + except Exception as e: + logger.error(f"Could not get results for query {query_name}: error: {e}") + continue ### push results to leaderboard API if query_results is None: - logger.error(f"Could not get results for query {query_name}") + logger.error(f"Could not get results for query {query_name} in time") continue - leaderboard_push_api_url = f"{args.engine_api}/leaderboard/{leaderboard_id}/scores?normalize_addresses={leaderboard_data['normalize_addresses']}" + leaderboard_push_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/scores?normalize_addresses={leaderboard_data['normalize_addresses']}" leaderboard_api_headers = { "Authorization": f"Bearer {args.query_api_access_token}", "Content-Type": "application/json", } - leaderboard_api_response = requests.put( - leaderboard_push_api_url, - json=query_results["data"], - headers=leaderboard_api_headers, - timeout=10, - ) + try: + leaderboard_api_response = requests.put( + leaderboard_push_api_url, + json=query_results["data"], + headers=leaderboard_api_headers, + timeout=10, + ) + except Exception as e: + logger.error( + f"Could not push results to leaderboard API: {e} for leaderboard {leaderboard_id}" + ) + continue try: leaderboard_api_response.raise_for_status() @@ -118,7 +140,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: ### get leaderboard from leaderboard API leaderboard_api_info_url = ( - f"{args.engine_api}/leaderboard/info?leaderboard_id={leaderboard_id}" + f"{MOONSTREAM_ENGINE_URL}/leaderboard/info?leaderboard_id={leaderboard_id}" ) leaderboard_api_response = requests.get( @@ -136,16 +158,16 @@ def handle_leaderboards(args: argparse.Namespace) -> None: info = leaderboard_api_response.json() logger.info( - f"Successfully pushed results to leaderboard {info['id']}: {info['title']}" + f"Successfully pushed results to leaderboard {info['id']}:{blue_c} {info['title']} {end_c}" ) logger.info( - f"can be check on:{args.engine_api}/leaderboard/?leaderboard_id={leaderboard_id}" + f"can be check on:{green_c} {MOONSTREAM_ENGINE_URL}/leaderboard/?leaderboard_id={leaderboard_id} {end_c}" ) def main(): """ - Generates an argument parser for the "autocorns judge" command. + CLI for generating leaderboards from Moonstream Query API """ parser = argparse.ArgumentParser(description="The Judge: Generate leaderboards") @@ -155,16 +177,6 @@ def main(): leaderboard_generator_parser = subparsers.add_parser( "leaderboards-generate", description="Generate Leaderboard" ) - leaderboard_generator_parser.add_argument( - "--query-api", - default="https://api.moonstream.to", - help="Moonstream API URL.", - ) - leaderboard_generator_parser.add_argument( - "--engine-api", - default="https://engineapi.moonstream.to", - help="Moonstream Engine API URL.", - ) leaderboard_generator_parser.add_argument( "--leaderboard-id", type=uuid.UUID, diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py index 8ec3013b..e4107f89 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -8,6 +8,8 @@ from typing import Any, Dict, Optional import requests # type: ignore +from ..settings import MOONSTREAM_API_URL + logging.basicConfig() logger = logging.getLogger(__name__) @@ -17,10 +19,32 @@ def get_results_for_moonstream_query( moonstream_access_token: str, query_name: str, params: Dict[str, Any], - api_url: str = "https://api.moonstream.to", + blockchain: Optional[str] = None, + api_url: str = MOONSTREAM_API_URL, max_retries: int = 100, interval: float = 30.0, ) -> Optional[Dict[str, Any]]: + """ + + Run update of query data and avaiting update of query result on S3. + TODO: Move to moonstream-client. + + :param moonstream_access_token: Moonstream access token. + + :param query_name: Name of the query to run. + + :param params: Parameters to pass to the query. + + :param api_url: URL of the Moonstream API. + + :param max_retries: Maximum number of times to retry getting results from the Moonstream Query API. + + :param interval: Number of seconds to wait between attempts to get results from the Moonstream Query API. + + :return: Results of the query. + + """ + result: Optional[Dict[str, Any]] = None api_url = api_url.rstrip("/") @@ -33,7 +57,10 @@ def get_results_for_moonstream_query( if_modified_since_datetime = datetime.datetime.utcnow() if_modified_since = if_modified_since_datetime.strftime("%a, %d %b %Y %H:%M:%S GMT") - request_body = {"params": params} + request_body: Dict[str, Any] = {"params": params} + + if blockchain is not None: + request_body["blockchain"] = blockchain success = False attempts = 0 diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index f6d43f39..c846a8c9 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -23,6 +23,11 @@ if MOONSTREAM_ENTITY_URL == "": entity_client = Entity(MOONSTREAM_ENTITY_URL) +MOONSTREAM_API_URL = os.environ.get("MOONSTREAM_API_URL", "https://api.moonstream.to") +MOONSTREAM_ENGINE_URL = os.environ.get( + "MOONSTREAM_ENGINE_URL", "https://engineapi.moonstream.to" +) + BUGOUT_REQUEST_TIMEOUT_SECONDS_RAW = os.environ.get( "MOONSTREAM_BUGOUT_TIMEOUT_SECONDS", 30 diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 57e91f94..c027fbe4 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -7,7 +7,11 @@ export HUMBUG_REPORTER_CRAWLERS_TOKEN="" # Entity environment variables export MOONSTREAM_ENTITY_URL="https://api.moonstream.to/entity" +# Engine environment variables +export MOONSTREAM_ENGINE_URL="https://engineapi.moonstream.to" + # Moonstream environment variables +export MOONSTREAM_API_URL="https://api.moonstream.to" export MOONSTREAM_BUGOUT_TIMEOUT_SECONDS=30 export MOONSTREAM_CORS_ALLOWED_ORIGINS="http://localhost:3000,https://moonstream.to,https://www.moonstream.to" export MOONSTREAM_DB_URI="postgresql://:@:/" From be7a7e026782bc8017a563c0154c08608aac3d09 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 27 Jul 2023 07:01:11 +0300 Subject: [PATCH 6/7] fix typo. --- crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py index e4107f89..fa235cca 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -26,7 +26,7 @@ def get_results_for_moonstream_query( ) -> Optional[Dict[str, Any]]: """ - Run update of query data and avaiting update of query result on S3. + Run update of query data and waiting update of query result on S3. TODO: Move to moonstream-client. :param moonstream_access_token: Moonstream access token. From f2f078b9c3f7d1873fa4971741d2253a674aaa0c Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 27 Jul 2023 07:06:14 +0300 Subject: [PATCH 7/7] add variable to sample.env --- crawlers/mooncrawl/sample.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index c027fbe4..c1db68a4 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -49,3 +49,6 @@ export MOONSTREAM_S3_PUBLIC_DATA_BUCKET="" export MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX="dev" export MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN="" export INFURA_PROJECT_ID="" + +# Leaderboard worker +export MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID= \ No newline at end of file