2023-07-26 20:52:35 +00:00
import argparse
import json
import logging
2023-08-01 08:58:36 +00:00
from typing import cast , List
2023-07-26 20:52:35 +00:00
import uuid
import requests # type: ignore
2023-08-01 08:58:36 +00:00
from bugout . data import BugoutSearchResult
2023-07-26 20:52:35 +00:00
2023-12-09 04:42:36 +00:00
from . utils import get_results_for_moonstream_query , leaderboard_push_batch
2023-07-26 20:52:35 +00:00
from . . settings import (
MOONSTREAM_ADMIN_ACCESS_TOKEN ,
MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID ,
2023-07-27 03:59:01 +00:00
MOONSTREAM_API_URL ,
MOONSTREAM_ENGINE_URL ,
2023-12-09 04:42:36 +00:00
MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE ,
MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS ,
2023-07-26 20:52:35 +00:00
)
from . . settings import bugout_client as bc
logger = logging . getLogger ( __name__ )
logger . setLevel ( logging . INFO )
2023-07-27 03:59:01 +00:00
blue_c = " \033 [94m "
green_c = " \033 [92m "
end_c = " \033 [0m "
2023-07-26 20:52:35 +00:00
def handle_leaderboards ( args : argparse . Namespace ) - > None :
"""
Run the leaderboard generator .
Get query from journal and push results to leaderboard API .
"""
### get leaderboard journal
2023-12-09 04:42:36 +00:00
leaderboard_push_batch_size = args . leaderboard_push_batch_size
leaderboard_push_timeout_seconds = args . leaderboard_push_timeout_seconds
2023-08-16 02:21:44 +00:00
query = " #leaderboard #status:active "
2023-07-26 20:52:35 +00:00
2023-12-09 04:42:36 +00:00
if args . leaderboard_id : # way to run only one leaderboard without status:active
query = f " #leaderboard #leaderboard_id: { args . leaderboard_id } "
2023-07-27 03:59:01 +00:00
try :
leaderboards = bc . search (
token = MOONSTREAM_ADMIN_ACCESS_TOKEN ,
journal_id = MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID ,
query = query ,
limit = 100 ,
2023-08-16 02:21:44 +00:00
timeout = 10 ,
2023-07-27 03:59:01 +00:00
)
2023-08-01 08:58:36 +00:00
leaderboards_results = cast ( List [ BugoutSearchResult ] , leaderboards . results )
2023-07-27 03:59:01 +00:00
except Exception as e :
logger . error ( f " Could not get leaderboards from journal: { e } " )
return
2023-07-26 20:52:35 +00:00
2023-08-01 08:58:36 +00:00
if len ( leaderboards_results ) == 0 :
2023-07-27 03:59:01 +00:00
logger . error ( " No leaderboard found " )
return
2023-07-26 20:52:35 +00:00
2023-08-01 08:58:36 +00:00
logger . info ( f " Found { len ( leaderboards_results ) } leaderboards " )
2023-07-26 20:52:35 +00:00
2023-08-01 08:58:36 +00:00
for leaderboard in leaderboards_results :
2023-07-26 20:52:35 +00:00
logger . info (
f " Processing leaderboard: { leaderboard . title } with id: { [ tag for tag in leaderboard . tags if tag . startswith ( ' leaderboard_id ' ) ] } "
)
if leaderboard . content is None :
continue
try :
leaderboard_data = json . loads ( leaderboard . content )
except json . JSONDecodeError :
logger . error (
2023-07-27 03:59:01 +00:00
f " Could not parse leaderboard content: { [ tag for tag in leaderboard . tags if tag . startswith ( ' leaderboard_id ' ) ] } in entry { leaderboard . entry_url . split ( ' / ' ) [ - 1 ] } "
2023-07-26 20:52:35 +00:00
)
continue
### get results from query API
leaderboard_id = leaderboard_data [ " leaderboard_id " ]
query_name = leaderboard_data [ " query_name " ]
if args . params :
params = json . loads ( args . params )
else :
params = leaderboard_data [ " params " ]
2023-07-27 03:59:01 +00:00
blockchain = leaderboard_data . get ( " blockchain " , None )
2023-07-26 20:52:35 +00:00
2023-07-27 03:59:01 +00:00
### execute query
try :
query_results = get_results_for_moonstream_query (
args . query_api_access_token ,
query_name ,
params ,
blockchain ,
MOONSTREAM_API_URL ,
args . max_retries ,
args . interval ,
2023-11-06 18:10:45 +00:00
args . query_api_retries ,
2023-07-27 03:59:01 +00:00
)
except Exception as e :
logger . error ( f " Could not get results for query { query_name } : error: { e } " )
continue
2023-07-26 20:52:35 +00:00
### push results to leaderboard API
if query_results is None :
2023-07-27 03:59:01 +00:00
logger . error ( f " Could not get results for query { query_name } in time " )
2023-07-26 20:52:35 +00:00
continue
2023-07-28 05:28:14 +00:00
leaderboard_push_api_url = f " { MOONSTREAM_ENGINE_URL } /leaderboard/ { leaderboard_id } /scores?normalize_addresses= { leaderboard_data [ ' normalize_addresses ' ] } &overwrite=true "
2023-07-26 20:52:35 +00:00
leaderboard_api_headers = {
" Authorization " : f " Bearer { args . query_api_access_token } " ,
" Content-Type " : " application/json " ,
}
2023-12-09 04:42:36 +00:00
if len ( query_results [ " data " ] ) > leaderboard_push_batch_size :
logger . info (
f " Pushing { len ( query_results [ ' data ' ] ) } scores to leaderboard { leaderboard_id } in batches of { leaderboard_push_batch_size } "
2023-07-27 03:59:01 +00:00
)
2023-12-09 04:42:36 +00:00
leaderboard_push_batch (
leaderboard_id ,
leaderboard_data ,
query_results [ " data " ] ,
leaderboard_api_headers ,
leaderboard_push_batch_size ,
timeout = leaderboard_push_timeout_seconds ,
2023-07-27 03:59:01 +00:00
)
2023-07-26 20:52:35 +00:00
2023-12-09 04:42:36 +00:00
else :
try :
leaderboard_api_response = requests . put (
leaderboard_push_api_url ,
json = query_results [ " data " ] ,
headers = leaderboard_api_headers ,
timeout = leaderboard_push_timeout_seconds ,
)
leaderboard_api_response . raise_for_status ( )
except requests . exceptions . HTTPError as http_error :
logger . error (
f " Could not push results to leaderboard API: { http_error . response . text } with status code { http_error . response . status_code } "
)
continue
2023-07-26 20:52:35 +00:00
### get leaderboard from leaderboard API
leaderboard_api_info_url = (
2023-07-27 03:59:01 +00:00
f " { MOONSTREAM_ENGINE_URL } /leaderboard/info?leaderboard_id= { leaderboard_id } "
2023-07-26 20:52:35 +00:00
)
leaderboard_api_response = requests . get (
leaderboard_api_info_url , headers = leaderboard_api_headers , timeout = 10
)
try :
leaderboard_api_response . raise_for_status ( )
except requests . exceptions . HTTPError as http_error :
logger . error (
f " Could not get leaderboard info from leaderboard API: { http_error . response . text } with status code { http_error . response . status_code } "
)
continue
info = leaderboard_api_response . json ( )
logger . info (
2023-07-27 03:59:01 +00:00
f " Successfully pushed results to leaderboard { info [ ' id ' ] } : { blue_c } { info [ ' title ' ] } { end_c } "
2023-07-26 20:52:35 +00:00
)
logger . info (
2023-07-27 03:59:01 +00:00
f " can be check on: { green_c } { MOONSTREAM_ENGINE_URL } /leaderboard/?leaderboard_id= { leaderboard_id } { end_c } "
2023-07-26 20:52:35 +00:00
)
def main ( ) :
"""
2023-07-27 03:59:01 +00:00
CLI for generating leaderboards from Moonstream Query API
2023-07-26 20:52:35 +00:00
"""
parser = argparse . ArgumentParser ( description = " The Judge: Generate leaderboards " )
parser . set_defaults ( func = lambda _ : parser . print_help ( ) )
subparsers = parser . add_subparsers ( )
leaderboard_generator_parser = subparsers . add_parser (
" leaderboards-generate " , description = " Generate Leaderboard "
)
leaderboard_generator_parser . add_argument (
" --leaderboard-id " ,
type = uuid . UUID ,
required = False ,
help = " Leaderboard ID on Engine API " ,
)
leaderboard_generator_parser . add_argument (
" --max-retries " ,
type = int ,
2023-11-01 11:22:20 +00:00
default = 12 ,
2023-07-26 20:52:35 +00:00
help = " Number of times to retry requests for Moonstream Query results " ,
)
2023-11-06 18:10:45 +00:00
leaderboard_generator_parser . add_argument (
" --query-api-retries " ,
type = int ,
default = 3 ,
help = " Number of times to retry updating Moonstream Query data " ,
)
2023-07-26 20:52:35 +00:00
leaderboard_generator_parser . add_argument (
" --interval " ,
type = float ,
2023-11-01 11:22:20 +00:00
default = 10.0 ,
2023-07-26 20:52:35 +00:00
help = " Number of seconds to wait between attempts to get results from Moonstream Query API " ,
)
leaderboard_generator_parser . add_argument (
" --params " ,
type = json . loads ,
required = False ,
help = " Parameters to pass to Moonstream Query API. Use together with --leaderboard-id " ,
)
leaderboard_generator_parser . add_argument (
" --query-api-access-token " ,
type = str ,
required = True ,
help = " Moonstream Access Token to use for Moonstream Query API requests " ,
)
2023-12-09 04:42:36 +00:00
leaderboard_generator_parser . add_argument (
" --leaderboard-push-batch-size " ,
type = int ,
default = MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE ,
help = " Number of scores to push to leaderboard API at once " ,
)
leaderboard_generator_parser . add_argument (
" --leaderboard-push-timeout-seconds " ,
type = int ,
default = MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS ,
help = " Timeout for leaderboard API requests " ,
)
2023-07-26 20:52:35 +00:00
leaderboard_generator_parser . set_defaults ( func = handle_leaderboards )
args = parser . parse_args ( )
args . func ( args )
if __name__ == " __main__ " :
main ( )