diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 046298f0..125ba580 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -49,6 +49,8 @@ POLYGON_STATE_CLEAN_SERVICE_FILE="polygon-state-clean.service" POLYGON_STATE_CLEAN_TIMER_FILE="polygon-state-clean.timer" POLYGON_METADATA_SERVICE_FILE="polygon-metadata.service" POLYGON_METADATA_TIMER_FILE="polygon-metadata.timer" +POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE="polygon-cu-reports-tokenonomics.service" +POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE="polygon-cu-reports-tokenonomics.timer" # Mumbai service files MUMBAI_SYNCHRONIZE_SERVICE="mumbai-synchronize.service" @@ -269,3 +271,12 @@ cp "${SCRIPT_DIR}/${POLYGON_METADATA_SERVICE_FILE}" "/etc/systemd/system/${POLYG cp "${SCRIPT_DIR}/${POLYGON_METADATA_TIMER_FILE}" "/etc/systemd/system/${POLYGON_METADATA_TIMER_FILE}" systemctl daemon-reload systemctl restart --no-block "${POLYGON_METADATA_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Polygon CU reports tokenonomics service and timer with: ${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}, ${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" +cp "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" "/etc/systemd/system/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" "/etc/systemd/system/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" +systemctl daemon-reload +systemctl restart --no-block "${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" diff --git a/crawlers/deploy/mumbai-missing.timer b/crawlers/deploy/mumbai-missing.timer index 0e7644a3..e3812f81 100644 --- a/crawlers/deploy/mumbai-missing.timer +++ b/crawlers/deploy/mumbai-missing.timer @@ -2,7 +2,7 @@ Description=Fill missing blocks at Mumbai database [Timer] -OnBootSec=30s +OnBootSec=120s OnUnitActiveSec=15m [Install] diff --git a/crawlers/deploy/polygon-cu-reports-tokenonomics.service b/crawlers/deploy/polygon-cu-reports-tokenonomics.service new file mode 100644 index 00000000..0393062e --- /dev/null +++ b/crawlers/deploy/polygon-cu-reports-tokenonomics.service @@ -0,0 +1,13 @@ +[Unit] +Description=Runs custom crawler for CU tokenonomics +After=network.target + +[Service] +Type=oneshot +User=ubuntu +Group=www-data +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.custom-crawler cu-reports --moonstream-token "${MOONSTREAM_ADMIN_ACCESS_TOKEN}" queries run-tokenonomics +CPUWeight=60 +SyslogIdentifier=polygon-cu-reports-tokenonomics diff --git a/crawlers/deploy/polygon-cu-reports-tokenonomics.timer b/crawlers/deploy/polygon-cu-reports-tokenonomics.timer new file mode 100644 index 00000000..3a56d85d --- /dev/null +++ b/crawlers/deploy/polygon-cu-reports-tokenonomics.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs custom crawler for CU tokenonomics + +[Timer] +OnBootSec=60s +OnUnitActiveSec=60m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/polygon-missing.timer b/crawlers/deploy/polygon-missing.timer index 58eb0ce3..73029113 100644 --- a/crawlers/deploy/polygon-missing.timer +++ b/crawlers/deploy/polygon-missing.timer @@ -2,7 +2,7 @@ Description=Fill missing blocks at Polygon database [Timer] -OnBootSec=30s +OnBootSec=120s OnUnitActiveSec=15m [Install] diff --git a/crawlers/deploy/xdai-missing.timer b/crawlers/deploy/xdai-missing.timer index 66285c66..d6ff5e2b 100644 --- a/crawlers/deploy/xdai-missing.timer +++ b/crawlers/deploy/xdai-missing.timer @@ -2,7 +2,7 @@ Description=Fill missing blocks at XDai database [Timer] -OnBootSec=35s +OnBootSec=120s OnUnitActiveSec=15m [Install] diff --git a/crawlers/mooncrawl/mooncrawl/contract/test_deployment_crawler.py b/crawlers/mooncrawl/mooncrawl/contract/test_deployment_crawler.py index 6568749b..21f1dbe4 100644 --- a/crawlers/mooncrawl/mooncrawl/contract/test_deployment_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/contract/test_deployment_crawler.py @@ -13,7 +13,7 @@ class TestDeploymentCrawler(TestCase): batch_size = 10 result = get_batch_block_range(from_block, to_block, batch_size) - last_end: Optional[int] = None + last_end: Optional[int] = None # type: ignore for batch_start, batch_end in result: if last_end is not None: self.assertEqual(batch_start, last_end + 1) @@ -30,7 +30,7 @@ class TestDeploymentCrawler(TestCase): batch_size = 10 result = get_batch_block_range(from_block, to_block, batch_size) - last_end: Optional[int] = None + last_end: Optional[int] = None # type: ignore for batch_start, batch_end in result: if last_end is not None: self.assertEqual(batch_start, last_end - 1) diff --git a/crawlers/mooncrawl/mooncrawl/cu_reports_crawler/__init__.py b/crawlers/mooncrawl/mooncrawl/cu_reports_crawler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/crawlers/mooncrawl/mooncrawl/cu_reports_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/cu_reports_crawler/cli.py new file mode 100644 index 00000000..109e4db0 --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/cu_reports_crawler/cli.py @@ -0,0 +1,638 @@ +import argparse +import datetime +from moonstream.client import Moonstream # type: ignore +import time +import requests +import json + +from typing import Any, Dict, Union + +from uuid import UUID + +from .queries import tokenomics_queries, cu_bank_queries + +from ..settings import ( + MOONSTREAM_S3_PUBLIC_DATA_BUCKET, + MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, +) + + +addresess_erc20_721 = { + "0x64060aB139Feaae7f06Ca4E63189D86aDEb51691": "ERC20", # UNIM + "0x431CD3C9AC9Fc73644BF68bF5691f4B83F9E104f": "ERC20", # RBW + "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f": "NFT", # unicorns + "0xA2a13cE1824F3916fC84C65e559391fc6674e6e8": "NFT", # lands + "0xa7D50EE3D7485288107664cf758E877a0D351725": "NFT", # shadowcorns +} + +addresess_erc1155 = ["0x99A558BDBdE247C2B2716f0D4cFb0E246DFB697D"] + + +def recive_S3_data_from_query( + client: Moonstream, + token: Union[str, UUID], + query_name: str, + params: Dict[str, Any], + time_await: int = 2, + max_retries: int = 20, +) -> Any: + + """ + Await the query to be update data on S3 with if_modified_since and return new the data. + """ + + keep_going = True + + repeat = 0 + + if_modified_since_datetime = datetime.datetime.utcnow() + if_modified_since = if_modified_since_datetime.strftime("%a, %d %b %Y %H:%M:%S GMT") + + time.sleep(2) + + data_url = client.exec_query( + token=token, + name=query_name, + params=params, + ) # S3 presign_url + + while keep_going: + time.sleep(time_await) + try: + data_response = requests.get( + data_url.url, + headers={"If-Modified-Since": if_modified_since}, + timeout=5, + ) + except Exception as e: + print(e) + continue + + if data_response.status_code == 200: + break + + repeat += 1 + + if repeat == max_retries // 2: + data_url = client.exec_query( + token=token, + name=query_name, + params=params, + ) + + if repeat > max_retries: + print("Too many retries") + break + return data_response.json() + + +def generate_report( + client: Moonstream, + token: Union[str, UUID], + query_name: str, + params: Dict[str, Any], + bucket_prefix: str, + bucket: str, + key: str, +): + """ + Generate the report. + """ + + try: + + json_data = recive_S3_data_from_query( + client=client, + token=token, + query_name=query_name, + params=params, + ) + + client.upload_query_results( + json.dumps(json_data), + bucket, + f"{bucket_prefix}/{key}", + ) + print( + f"Report generated and results uploaded at: https://{bucket}/{bucket_prefix}/{key}" + ) + except Exception as err: + print( + f"Cant recive or load data for s3, for query: {query_name}, bucket: {bucket}, key: {key}. End with error: {err}" + ) + + +def create_user_query( + client: Moonstream, + token: Union[str, UUID], + query_name: str, + query: str, +): + """ + Create a user query. + """ + + try: + client.create_query(token=token, name=query_name, query=query) + except Exception as err: + print(f"Cant create user query: {query_name}. End with error: {err}") + + +def delete_user_query(client: Moonstream, token: str, query_name: str): + """ + Delete the user's queries. + """ + + id = client.delete_query( + token=token, + name=query_name, + ) + + print(f"Query with name:{query_name} and id: {id} was deleted") + + +def init_game_bank_queries_handler(args: argparse.Namespace): + + """ + Create the game bank queries. + """ + + client = Moonstream() + + for query in cu_bank_queries: + + try: + if args.overwrite: + try: + # delete + delete_user_query( + client=client, + token=args.moonstream_token, + query_name=query["name"], + ) + except Exception as err: + print(err) + # create + created_entry = client.create_query( + token=args.moonstream_token, + name=query["name"], + query=query["query"], + ) + print( + f"Created query {query['name']} please validate it in the UI url {created_entry.journal_url}/entries/{created_entry.id}/" + ) + except Exception as e: + print(e) + pass + + +def init_tokenomics_queries_handler(args: argparse.Namespace): + + """ + Create the tokenomics queries. + """ + + client = Moonstream() + + for query in tokenomics_queries: + + try: + if args.overwrite: + try: + # delete + delete_user_query( + client=client, + token=args.moonstream_token, + query_name=query["name"], + ) + except Exception as err: + print(err) + # create + created_entry = client.create_query( + token=args.moonstream_token, + name=query["name"], + query=query["query"], + ) + print( + f"Created query {query['name']} please validate it in the UI url {created_entry.journal_url}/entries/{created_entry.id}/" + ) + except Exception as e: + print(e) + pass + + +def run_tokenomics_queries_handler(args: argparse.Namespace): + + client = Moonstream() + + query_name = "erc20_721_volume" + + ### Run voluem query + + ranges = [ + {"time_format": "YYYY-MM-DD HH24", "time_range": "24 hours"}, + {"time_format": "YYYY-MM-DD HH24", "time_range": "7 days"}, + {"time_format": "YYYY-MM-DD", "time_range": "30 days"}, + ] + + # volume of erc20 and erc721 + + for address, type in addresess_erc20_721.items(): + for range in ranges: + + params: Dict[str, Any] = { + "address": address, + "type": type, + "time_format": range["time_format"], + "time_range": range["time_range"], + } + + generate_report( + client=client, + token=args.moonstream_token, + query_name=query_name, + params=params, + bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, + bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET, + key=f'{query_name}/{address}/{range["time_range"].replace(" ","_")}/data.json', + ) + + # volume change of erc20 and erc721 + + query_name = "volume_change" + + for address, type in addresess_erc20_721.items(): + for range in ranges: + + params = { + "address": address, + "type": type, + "time_range": range["time_range"], + } + + generate_report( + client=client, + token=args.moonstream_token, + query_name=query_name, + params=params, + bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, + bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET, + key=f'{query_name}/{address}/{range["time_range"].replace(" ","_")}/data.json', + ) + + query_name = "erc1155_volume" + + # volume of erc1155 + + addresess_erc1155 = ["0x99A558BDBdE247C2B2716f0D4cFb0E246DFB697D"] + + for address in addresess_erc1155: + for range in ranges: + + params = { + "address": address, + "time_format": range["time_format"], + "time_range": range["time_range"], + } + + generate_report( + client=client, + token=args.moonstream_token, + query_name=query_name, + params=params, + bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, + bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET, + key=f"{query_name}/{address}/{range['time_range'].replace(' ','_')}/data.json", + ) + + # most_recent_sale + + query_name = "most_recent_sale" + + for address, type in addresess_erc20_721.items(): + if type == "NFT": + for amount in [10, 100]: + params = { + "address": address, + "amount": amount, + } + + generate_report( + client=client, + token=args.moonstream_token, + query_name=query_name, + params=params, + bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, + bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET, + key=f"{query_name}/{address}/{amount}/data.json", + ) + + # most_active_buyers + + query_name = "most_active_buyers" + + for address, type in addresess_erc20_721.items(): + + if type == "NFT": + + for range in ranges: + + params = { + "address": address, + "time_range": range["time_range"], + } + + generate_report( + client=client, + token=args.moonstream_token, + query_name=query_name, + params=params, + bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, + bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET, + key=f"{query_name}/{address}/{range['time_range'].replace(' ','_')}/data.json", + ) + + # most_active_sellers + + query_name = "most_active_sellers" + + for address, type in addresess_erc20_721.items(): + + if type == "NFT": + + for range in ranges: + + params = { + "address": address, + "time_range": range["time_range"], + } + + generate_report( + client=client, + token=args.moonstream_token, + query_name=query_name, + params=params, + bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, + bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET, + key=f"{query_name}/{address}/{range['time_range'].replace(' ','_')}/data.json", + ) + + # lagerst_owners + + query_name = "lagerst_owners" + for address, type in addresess_erc20_721.items(): + + if type == "NFT": + + params = { + "address": address, + } + + generate_report( + client=client, + token=args.moonstream_token, + query_name=query_name, + params=params, + bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, + bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET, + key=f"{query_name}/{address}/data.json", + ) + + # total_supply_erc721 + + query_name = "total_supply_erc721" + + for address, type in addresess_erc20_721.items(): + + if type == "NFT": + + params = { + "address": address, + } + + generate_report( + client=client, + token=args.moonstream_token, + query_name=query_name, + params=params, + bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, + bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET, + key=f"{query_name}/{address}/data.json", + ) + + # total_supply_terminus + + query_name = "total_supply_terminus" + + for address in addresess_erc1155: + + params = { + "address": address, + } + + generate_report( + client=client, + token=args.moonstream_token, + query_name=query_name, + params=params, + bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, + bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET, + key=f"{query_name}/{address}/data.json", + ) + + print("Done") + + +def list_user_queries_handler(args: argparse.Namespace): + """ + List the user's queries. + """ + + client = Moonstream() + + queries = client.list_queries( + token=args.moonstream_token, + ) + + for query in queries.queries: + print(query.name, query.id) + + +def delete_user_query_handler(args: argparse.Namespace): + """ + Delete the user's queries. + """ + client = Moonstream() + + delete_user_query(client=client, token=args.moonstream_token, query_name=args.name) + + +def create_user_query_handler(args: argparse.Namespace): + """ + Create the user's queries. + """ + client = Moonstream() + + for query in tokenomics_queries: + + if query["name"] == args.name: + + create_user_query( + client=client, + token=args.moonstream_token, + query_name=query["name"], + query=query["query"], + ) + + +def generate_game_bank_report(args: argparse.Namespace): + """ + han + Generate the game bank query. + """ + + client = Moonstream() + + for query in client.list_queries( + token=args.moonstream_token, + ).queries: + + params = {} + + if ( + query.name == "cu-bank-withdrawals-total" + or query.name == "cu-bank-withdrawals-events" + ): + blocktimestamp = int(time.time()) + params = {"block_timestamp": blocktimestamp} + + keep_going = True + + if_modified_since_datetime = datetime.datetime.utcnow() + if_modified_since = if_modified_since_datetime.strftime( + "%a, %d %b %Y %H:%M:%S GMT" + ) + + data_url = client.exec_query( + token=args.moonstream_token, + query_name=query.name, + params=params, + ) # S3 presign_url + while keep_going: + data_response = requests.get( + data_url, + headers={"If-Modified-Since": if_modified_since}, + timeout=10, + ) + # push to s3 + + if data_response.status_code == 200: + json.dumps(data_response.json()) + break + else: + time.sleep(2) + continue + + pass + + +def main(): + + parser = argparse.ArgumentParser() + + parser.set_defaults(func=lambda _: parser.print_help()) + + subparsers = parser.add_subparsers() + + cu_reports_parser = subparsers.add_parser("cu-reports", help="CU Reports") + + cu_reports_subparsers = cu_reports_parser.add_subparsers() + + cu_reports_parser.add_argument( + "--moonstream-token", + required=True, + type=str, + ) + queries_parser = cu_reports_subparsers.add_parser( + "queries", + help="Queries commands", + ) + + queries_parser.set_defaults(func=lambda _: queries_parser.print_help()) + + queries_subparsers = queries_parser.add_subparsers() + + queries_subparsers.add_parser( + "list", + help="List all queries", + description="List all queries", + ).set_defaults(func=list_user_queries_handler) + + init_game_bank_parser = queries_subparsers.add_parser( + "init-game-bank", + help="Create all predifind query", + description="Create all predifind query", + ) + + init_game_bank_parser.add_argument("--overwrite", type=bool, default=False) + + init_game_bank_parser.set_defaults(func=init_game_bank_queries_handler) + + init_tokenonomics_parser = queries_subparsers.add_parser( + "init-tokenonomics", + help="Create all predifind query", + description="Create all predifind query", + ) + + init_tokenonomics_parser.add_argument("--overwrite", type=bool, default=False) + + init_tokenonomics_parser.set_defaults(func=init_tokenomics_queries_handler) + + generate_report = queries_subparsers.add_parser( + "run-tokenonomics", + help="Run tokenomics queries and push to S3 public backet", + description="Run tokenomics queries and push to S3 public backet", + ) + + generate_report.set_defaults(func=run_tokenomics_queries_handler) + + delete_query = queries_subparsers.add_parser( + "delete", + help="Delete all predifind query", + description="Delete all predifind query", + ) + + delete_query.add_argument( + "--name", + required=True, + type=str, + ) + + delete_query.set_defaults(func=delete_user_query_handler) + + create_query = queries_subparsers.add_parser( + "create", + help="Create all predifind query", + description="Create all predifind query", + ) + + create_query.add_argument( + "--name", + required=True, + type=str, + ) + + create_query.set_defaults(func=create_user_query_handler) + + cu_bank_parser = cu_reports_subparsers.add_parser( + "generate-reports", + help="Generate cu-bank state reports", + ) + + cu_bank_parser.set_defaults(func=generate_game_bank_report) + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/crawlers/mooncrawl/mooncrawl/cu_reports_crawler/queries.py b/crawlers/mooncrawl/mooncrawl/cu_reports_crawler/queries.py new file mode 100644 index 00000000..4f6a0707 --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/cu_reports_crawler/queries.py @@ -0,0 +1,701 @@ +cu_bank_queries = [ + { + "name": "cu-bank-blances", + "query": """ + WITH game_contract as ( + SELECT + * + from + polygon_labels + where + address = '0x94f557dDdb245b11d031F57BA7F2C4f28C4A203e' + and label = 'moonworm-alpha' + ) + SELECT + address, + div(sum( + CASE + WHEN result_balances.token_address = '0x64060aB139Feaae7f06Ca4E63189D86aDEb51691' THEN amount + ELSE 0 + END + ), 10^18::decimal) as UNIM_BALANCE, + div(sum( + CASE + WHEN result_balances.token_address = '0x431CD3C9AC9Fc73644BF68bF5691f4B83F9E104f' THEN amount + ELSE 0 + END + ), 10^18::decimal) as RBW_BALANCE + FROM + ( + select + transaction_hash, + label_data -> 'args' ->> 'player' as address, + jsonb_array_elements(label_data -> 'args' -> 'tokenAddresses') ->> 0 as token_address, + - jsonb_array_elements(label_data -> 'args' -> 'tokenAmounts') :: decimal as amount + from + game_contract + where + label_data ->> 'name' = 'UnstashedMultiple' + union + ALL + select + transaction_hash, + label_data -> 'args' ->> 'player' as address, + label_data -> 'args' ->> 'token' as token_address, + -((label_data -> 'args' -> 'amount') :: decimal) as amount + from + game_contract + where + label_data ->> 'name' = 'Unstashed' + union + ALL + select + transaction_hash, + label_data -> 'args' ->> 'player' as address, + label_data -> 'args' ->> 'token' as token_address, + (label_data -> 'args' ->> 'amount') :: decimal as amount + from + game_contract + where + label_data ->> 'name' = 'Stashed' + union + ALL + select + transaction_hash, + label_data -> 'args' ->> 'player' as address, + jsonb_array_elements(label_data -> 'args' -> 'tokenAddresses') ->> 0 as token_address, + jsonb_array_elements(label_data -> 'args' -> 'tokenAmounts') :: decimal as amount + from + game_contract + where + label_data ->> 'name' = 'StashedMultiple' + + ) result_balances + group by + address + ORDER BY + UNIM_BALANCE DESC, + RBW_BALANCE DESC +""", + }, + { + "name": "cu-bank-withdrawals-total", + "query": """ + WITH game_contract as ( + SELECT + * + from + polygon_labels + where + address = '0x94f557dDdb245b11d031F57BA7F2C4f28C4A203e' + and label = 'moonworm-alpha' + block_timestamp >= :block_timestamp + ), withdoraws_total as ( + SELECT + address, + div(sum( + CASE + WHEN result_balances.token_address = '0x64060aB139Feaae7f06Ca4E63189D86aDEb51691' THEN amount + ELSE 0 + END + ), 10^18::decimal) as UNIM_BALANCE, + div(sum( + CASE + WHEN result_balances.token_address = '0x431CD3C9AC9Fc73644BF68bF5691f4B83F9E104f' THEN amount + ELSE 0 + END + ), 10^18::decimal) as RBW_BALANCE + FROM + ( + select + transaction_hash, + label_data -> 'args' ->> 'player' as address, + jsonb_array_elements(label_data -> 'args' -> 'tokenAddresses') ->> 0 as token_address, + jsonb_array_elements(label_data -> 'args' -> 'tokenAmounts') :: decimal as amount + from + game_contract + where + label_data ->> 'name' = 'UnstashedMultiple' + union + ALL + select + transaction_hash, + label_data -> 'args' ->> 'player' as address, + label_data -> 'args' ->> 'token' as token_address, + ((label_data -> 'args' -> 'amount') :: decimal) as amount + from + game_contract + where + label_data ->> 'name' = 'Unstashed' + ) result_balances + group by + address + ORDER BY + UNIM_BALANCE DESC, + RBW_BALANCE DESC + ) + SELECT + address, + UNIM_BALANCE, + RBW_BALANCE, + UNIM_BALANCE + RBW_BALANCE as TOTAL + FROM + withdoraws_total + ORDER BY + TOTAL DESC; +""", + }, + { + "name": "cu-bank-withdrawals-events", + "query": """ + WITH game_contract as ( + SELECT + * + from + polygon_labels + where + address = '0x94f557dDdb245b11d031F57BA7F2C4f28C4A203e' + and label = 'moonworm-alpha' + block_timestamp >= :block_timestamp + ), withdoraws_total as ( + SELECT + address, + CASE + WHEN result_balances.token_address = '0x64060aB139Feaae7f06Ca4E63189D86aDEb51691' THEN 'UNIM' + WHEN result_balances.token_address = '0x431CD3C9AC9Fc73644BF68bF5691f4B83F9E104f' THEN 'RBW' + END as currency, + div(amount, 10^18::decimal) as amount + FROM + ( + select + transaction_hash, + label_data -> 'args' ->> 'player' as address, + jsonb_array_elements(label_data -> 'args' -> 'tokenAddresses') ->> 0 as token_address, + jsonb_array_elements(label_data -> 'args' -> 'tokenAmounts') :: decimal as amount + from + game_contract + where + label_data ->> 'name' = 'UnstashedMultiple' + union + ALL + select + transaction_hash, + label_data -> 'args' ->> 'player' as address, + label_data -> 'args' ->> 'token' as token_address, + ((label_data -> 'args' -> 'amount') :: decimal) as amount + from + game_contract + where + label_data ->> 'name' = 'Unstashed' + ) result_balances + ) + SELECT + address, + currency, + amount, + FROM + withdoraws_total + ORDER BY + amount DESC + """, + }, +] + + +tokenomics_queries = [ + { + "name": "volume_change", + "query": """ + with all_transfers as ( + select + transaction_hash, + CASE + WHEN type: ='NFT' THEN 1 + ELSE (label_data->'args'->>'value')::decimal + END as value, + block_timestamp as block_timestamp + from polygon_labels + where label='moonworm-alpha' + and address= :address + and label_data->>'name'='Transfer' + ), after_range_transfer as ( + select + * + FROM + all_transfers + where block_timestamp >= extract(epoch from now() - interval :time_range)::int + ), current_volume as ( + SELECT + sum(all_transfers.value) as value, + sum( + CASE + WHEN to_address in ('0xF715bEb51EC8F63317d66f491E37e7BB048fCc2d','0xfede379e48C873C75F3cc0C81F7C784aD730a8F7','0x00000000006c3852cbef3e08e8df289169ede581') + THEN 1 + else 0 + END + ) as os_sales + from all_transfers + LEFT JOIN polygon_transactions ON all_transfers.transaction_hash = polygon_transactions.hash + ), volume_different as ( + select + sum(after_range_transfer.value) as value, + sum( + CASE + WHEN to_address in ('0xF715bEb51EC8F63317d66f491E37e7BB048fCc2d','0xfede379e48C873C75F3cc0C81F7C784aD730a8F7','0x00000000006c3852cbef3e08e8df289169ede581') + THEN 1 + else 0 + END + ) as os_sales + from after_range_transfer + LEFT JOIN polygon_transactions ON after_range_transfer.transaction_hash = polygon_transactions.hash + ) + SELECT + volume_different.value as diff, + volume_different.os_sales as os_diff, + current_volume.value as current, + current_volume.os_sales as os_current + from current_volume, volume_different + """, + }, + { + "name": "erc20_721_volume", + "query": """ + with interval_transfers as ( + select + transaction_hash, + CASE + WHEN :type ='NFT' THEN 1 + ELSE (label_data->'args'->>'value')::decimal + END as value, + label_data->'args'->>'to' as buyer, + label_data->'args'->>'from' as seller, + to_char(to_timestamp(block_timestamp), :time_format) as time + from polygon_labels + where label='moonworm-alpha' + and address= :address + and label_data->>'name'='Transfer' + and block_timestamp >= extract(epoch from now() - interval :time_range)::int + ) + SELECT + time as time, + sum(interval_transfers.value) as value, + sum( + CASE + WHEN to_address in ('0xF715bEb51EC8F63317d66f491E37e7BB048fCc2d','0xfede379e48C873C75F3cc0C81F7C784aD730a8F7','0x00000000006c3852cbef3e08e8df289169ede581') + THEN 1 + else 0 + END + ) as os_sales + from interval_transfers + LEFT JOIN polygon_transactions ON interval_transfers.transaction_hash = polygon_transactions.hash + GROUP BY time + """, + }, + { + "name": "erc1155_volume", + "query": """ + with labels_data as ( + select + * + from + polygon_labels + where address= :address + AND label='moonworm-alpha' + AND block_timestamp >= extract(epoch from now() - interval :time_range)::int + ), + nfts_data as ( + select + transaction_hash, + label_data->'args'->>'to' as buyer, + label_data->'args'->>'from' as seller, + jsonb_array_elements(label_data -> 'args' -> 'values') :: decimal as value, + jsonb_array_elements(label_data -> 'args' -> 'ids')->>0 as token_id, + to_char(to_timestamp(block_timestamp), :time_format) as time + from + labels_data + where + label_data ->> 'name' = 'TransferBatch' + UNION ALL + select + transaction_hash, + label_data->'args'->>'to' as buyer, + label_data->'args'->>'from' as seller, + (label_data -> 'args' ->> 'value') :: decimal as value, + label_data -> 'args' ->> 'id' as token_id, + to_char(to_timestamp(block_timestamp), :time_format) as time + from + labels_data + where + label_data ->> 'name' = 'TransferSingle' + ) + SELECT + time as time, + token_id as token_id, + sum(nfts_data.value) as value, + sum( + CASE + WHEN to_address in ('0xF715bEb51EC8F63317d66f491E37e7BB048fCc2d','0xfede379e48C873C75F3cc0C81F7C784aD730a8F7','0x00000000006c3852cbef3e08e8df289169ede581') + THEN 1 + else 0 + END + ) as os_sales + from nfts_data + LEFT JOIN polygon_transactions ON nfts_data.transaction_hash = polygon_transactions.hash + GROUP BY + time, + token_id + ORDER BY token_id::int, time DESC +""", + }, + { + "name": "most_recent_sale", + "query": """ + with contract_erc721_transfers as ( + select + transaction_hash, + label_data->'args'->>'tokenId' as token_id, + label_data->'args'->>'to' as buyer, + label_data->'args'->>'from' as seller, + block_timestamp as block_timestamp + from polygon_labels + where label='moonworm-alpha' + and address= :address + and label_data->>'name'='Transfer' + order by block_number desc + + ) + SELECT + polygon_transactions.hash as transaction_hash, + contract_erc721_transfers.block_timestamp as block_timestamp, + contract_erc721_transfers.token_id as token_id, + contract_erc721_transfers.buyer as buyer, + contract_erc721_transfers.seller as seller + from polygon_transactions + inner JOIN contract_erc721_transfers ON contract_erc721_transfers.transaction_hash = polygon_transactions.hash + where polygon_transactions.to_address in ('0xF715bEb51EC8F63317d66f491E37e7BB048fCc2d','0xfede379e48C873C75F3cc0C81F7C784aD730a8F7', '0x00000000006c3852cbef3e08e8df289169ede581') + limit :amount +""", + }, + { + "name": "most_active_buyers", + "query": """ + with contracts_data as ( + select + * + from polygon_labels + where label='moonworm-alpha' + and address= :address + and block_timestamp >= extract(epoch from now() - interval :time_range)::int + ), contract_nfts_transfers as ( + select + transaction_hash, + label_data->'args'->>'to' as buyer, + label_data->'args'->>'from' as seller + from contracts_data + where label_data->>'name'='Transfer' + UNION ALL + select + transaction_hash, + label_data->'args'->>'to' as buyer, + label_data->'args'->>'from' as seller + from + contracts_data + where + label_data ->> 'name' = 'TransferBatch' + UNION ALL + select + transaction_hash, + label_data->'args'->>'to' as buyer, + label_data->'args'->>'from' as seller + from + contracts_data + where + label_data ->> 'name' = 'TransferSingle' + + ) + SELECT + contract_nfts_transfers.buyer as buyer, + count(*) as sale_count + from polygon_transactions + inner JOIN contract_nfts_transfers ON contract_nfts_transfers.transaction_hash = polygon_transactions.hash + where polygon_transactions.to_address in ('0xF715bEb51EC8F63317d66f491E37e7BB048fCc2d','0xfede379e48C873C75F3cc0C81F7C784aD730a8F7','0x00000000006c3852cbEf3e08E8dF289169EdE581') + group by contract_nfts_transfers.buyer + order by sale_count desc + """, + }, + { + "name": "most_active_sellers", + "query": """ + with contracts_data as ( + select + * + from polygon_labels + where label='moonworm-alpha' + and address= :address + and block_timestamp >= extract(epoch from now() - interval :time_range)::int + ), contract_nfts_transfers as ( + select + transaction_hash, + label_data->'args'->>'to' as buyer, + label_data->'args'->>'from' as seller + from contracts_data + where label_data->>'name'='Transfer' + UNION ALL + select + transaction_hash, + label_data->'args'->>'to' as buyer, + label_data->'args'->>'from' as seller + from + contracts_data + where + label_data ->> 'name' = 'TransferBatch' + UNION ALL + select + transaction_hash, + label_data->'args'->>'to' as buyer, + label_data->'args'->>'from' as seller + from + contracts_data + where + label_data ->> 'name' = 'TransferSingle' + + ) + SELECT + contract_nfts_transfers.seller as seller, + count(*) as sale_count + from polygon_transactions + inner JOIN contract_nfts_transfers ON contract_nfts_transfers.transaction_hash = polygon_transactions.hash + where polygon_transactions.to_address in ('0xF715bEb51EC8F63317d66f491E37e7BB048fCc2d','0xfede379e48C873C75F3cc0C81F7C784aD730a8F7','0x00000000006c3852cbEf3e08E8dF289169EdE581') + group by contract_nfts_transfers.seller + order by sale_count desc + """, + }, + { + "name": "lagerst_owners", + "query": """ + WITH erc_1155_721_contracts_transfers_with_trashhold as ( + SELECT + label_data as label_data, + block_timestamp as block_timestamp, + address as address + from + polygon_labels + WHERE + polygon_labels.label = 'moonworm-alpha' + AND polygon_labels.address = :address + + ), + own_erc_1155_721_count as ( + Select + difference.address, + ( + difference.transfers_in - difference.transfers_out + ) as owned_nfts + from + ( + SELECT + total.address, + sum(total.transfer_out) as transfers_out, + sum(total.transfer_in) as transfers_in + from + ( + SELECT + label_data -> 'args' ->> 'from' as address, + jsonb_array_elements(label_data -> 'args' -> 'values') :: decimal as transfer_out, + 0 as transfer_in + from + erc_1155_721_contracts_transfers_with_trashhold + where + label_data ->> 'name' = 'TransferBatch' + UNION + ALL + SELECT + label_data -> 'args' ->> 'from' as address, + (label_data -> 'args' ->> 'value') :: decimal as transfer_out, + 0 as transfer_in + from + erc_1155_721_contracts_transfers_with_trashhold + where + label_data ->> 'name' = 'TransferSingle' + UNION + ALL + select + label_data -> 'args' ->> 'to' as address, + 0 as transfer_out, + (label_data -> 'args' ->>'value') :: decimal as transfer_in + from + erc_1155_721_contracts_transfers_with_trashhold + where + label_data ->> 'name' = 'TransferSingle' + UNION + ALL + select + label_data -> 'args' ->> 'to' as address, + 0 as transfer_out, + jsonb_array_elements(label_data -> 'args' -> 'values') :: decimal as transfer_in + from + erc_1155_721_contracts_transfers_with_trashhold + where + label_data ->> 'name' = 'TransferBatch' + UNION + ALL + select + label_data -> 'args' ->> 'from' as address, + 1 as transfer_out, + 0 as transfer_in + from + erc_1155_721_contracts_transfers_with_trashhold + + where + label_data ->> 'name' = 'Transfer' + UNION + ALL + select + label_data -> 'args' ->> 'to' as address, + 0 as transfer_out, + 1 as transfer_in + from + erc_1155_721_contracts_transfers_with_trashhold + + where + label_data ->> 'name' = 'Transfer' + ) as total + group by + address + ) difference + order by + owned_nfts desc + ) + SELECT + * + from + own_erc_1155_721_count + WHERE + address not in ( + '0x000000000000000000000000000000000000dEaD', + '0x0000000000000000000000000000000000000000' + ) + order by + owned_nfts desc + """, + }, + { + "name": "total_supply_erc721", + "query": """ + select + count(*) as total_supply + from( + SELECT DISTINCT ON((label_data->'args'->>'tokenId')::INT) (label_data->'args'->>'tokenId')::INT as token_id, + label_data->'args'->>'to' as current_owner + FROM polygon_labels + WHERE address = :address + AND (label = 'moonworm' or label = 'moonworm-alpha') + AND block_number >= 21418707 + AND label_data->>'type' = 'event' + AND label_data->>'name' = 'Transfer' + AND label_data->'args'->>'to' != '0x8d528e98A69FE27b11bb02Ac264516c4818C3942' + AND label_data->'args'->>'from' != '0x8d528e98A69FE27b11bb02Ac264516c4818C3942' + ORDER BY (label_data->'args'->>'tokenId')::INT ASC, + block_number::INT DESC, + log_index::INT DESC + ) as total_supply + where current_owner not in ('0x000000000000000000000000000000000000dEaD','0x0000000000000000000000000000000000000000') + """, + }, + { + "name": "total_supply_terminus", + "query": """ + WITH erc_1155_721_contracts_transfers_with_trashhold as ( + SELECT + label_data as label_data, + block_timestamp as block_timestamp, + address as address + from + polygon_labels + WHERE + polygon_labels.label = 'moonworm-alpha' + AND polygon_labels.address = :address + ), + own_erc_1155_721_count as ( + Select + + difference.address, + token_id, + ( + difference.transfers_in - difference.transfers_out + ) as owned_nfts + from + ( + SELECT + total.address, + token_id, + sum(total.transfer_out) as transfers_out, + sum(total.transfer_in) as transfers_in + from + ( + SELECT + label_data -> 'args' ->> 'from' as address, + jsonb_array_elements(label_data -> 'args' -> 'ids')->>0 as token_id, + jsonb_array_elements(label_data -> 'args' -> 'values') :: decimal as transfer_out, + 0 as transfer_in + from + erc_1155_721_contracts_transfers_with_trashhold + where + label_data ->> 'name' = 'TransferBatch' + UNION + ALL + SELECT + label_data -> 'args' ->> 'from' as address, + label_data -> 'args' ->> 'id' as token_id, + (label_data -> 'args' ->> 'value') :: decimal as transfer_out, + 0 as transfer_in + from + erc_1155_721_contracts_transfers_with_trashhold + where + label_data ->> 'name' = 'TransferSingle' + UNION + ALL + select + label_data -> 'args' ->> 'to' as address, + label_data -> 'args' ->> 'id' as token_id, + 0 as transfer_out, + (label_data -> 'args' ->>'value') :: decimal as transfer_in + from + erc_1155_721_contracts_transfers_with_trashhold + where + label_data ->> 'name' = 'TransferSingle' + UNION + ALL + select + label_data -> 'args' ->> 'to' as address, + jsonb_array_elements(label_data -> 'args' -> 'ids')->>0 as token_id, + 0 as transfer_out, + jsonb_array_elements(label_data -> 'args' -> 'values') :: decimal as transfer_in + from + erc_1155_721_contracts_transfers_with_trashhold + where + label_data ->> 'name' = 'TransferBatch' + ) as total + group by + address, token_id + ) difference + order by + owned_nfts desc + ) + SELECT + token_id as token_id, + sum(owned_nfts) as total_supply + from + own_erc_1155_721_count + WHERE + address not in ( + '0x000000000000000000000000000000000000dEaD', + '0x0000000000000000000000000000000000000000' + ) + group by + token_id + order by + token_id::int desc + """, + }, +] diff --git a/crawlers/mooncrawl/mooncrawl/middleware.py b/crawlers/mooncrawl/mooncrawl/middleware.py index a722df0c..b6995e0e 100644 --- a/crawlers/mooncrawl/mooncrawl/middleware.py +++ b/crawlers/mooncrawl/mooncrawl/middleware.py @@ -19,7 +19,7 @@ class MoonstreamHTTPException(HTTPException): status_code: int, detail: Any = None, headers: Optional[Dict[str, Any]] = None, - internal_error: Exception = None, + internal_error: Optional[Exception] = None, ): super().__init__(status_code, detail, headers) if internal_error is not None: diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index dd4d0bca..a1561d96 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -156,7 +156,7 @@ def get_crawl_job_entries( subscription_type: SubscriptionTypes, crawler_type: str, journal_id: str = MOONSTREAM_MOONWORM_TASKS_JOURNAL, - created_at_filter: int = None, + created_at_filter: Optional[int] = None, limit: int = 200, ) -> List[BugoutSearchResult]: """ diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 1cb64073..66e50dd6 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -193,3 +193,20 @@ multicall_contracts: Dict[AvailableBlockchainType, str] = { AvailableBlockchainType.MUMBAI: "0xe9939e7Ea7D7fb619Ac57f648Da7B1D425832631", AvailableBlockchainType.ETHEREUM: "0x5BA1e12693Dc8F9c48aAD8770482f4739bEeD696", } + + +# Custom Crawler + +MOONSTREAM_S3_PUBLIC_DATA_BUCKET = os.environ.get( + "MOONSTREAM_S3_PUBLIC_DATA_BUCKET", "" +) # S3 bucket for storing custom crawler data + +if MOONSTREAM_S3_PUBLIC_DATA_BUCKET == "": + raise ValueError( + "MOONSTREAM_S3_PUBLIC_DATA_BUCKET environment variable must be set" + ) + + +MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX = os.environ.get( + "MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX", "dev" +) diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index 99d4f330..b1322bcb 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.2.4" +MOONCRAWL_VERSION = "0.2.5" diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 23ee4108..1b28a874 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -31,3 +31,8 @@ export MOONSTREAM_S3_QUERIES_BUCKET_PREFIX="dev" # 3rd parties environment variables export MOONSTREAM_ETHERSCAN_TOKEN="" export COINMARKETCAP_API_KEY="" + + +# Custom crawler +export MOONSTREAM_S3_PUBLIC_DATA_BUCKET="" +export MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX="dev" \ No newline at end of file diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index f6282b7c..f0c1c219 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -38,6 +38,7 @@ setup( "chardet", "fastapi", "moonstreamdb>=0.3.2", + "moonstream>=0.1.1", "moonworm[moonstream]>=0.5.2", "humbug", "pydantic", @@ -64,6 +65,7 @@ setup( "statistics=mooncrawl.stats_worker.dashboard:main", "state-crawler=mooncrawl.state_crawler.cli:main", "metadata-crawler=mooncrawl.metadata_crawler.cli:main", + "custom-crawler=mooncrawl.cu_reports_crawler.cli:main", ] }, )