moonstream/crawlers/mooncrawl/mooncrawl/reports_crawler/cli.py

import argparse
import csv
import datetime
import json
import logging
import time
from io import StringIO
from typing import Any, Dict, Union
from uuid import UUID

import requests  # type: ignore
from moonstream.client import Moonstream  # type: ignore

from ..actions import recive_S3_data_from_query
from ..settings import (
    MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
    MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
)
from .queries import cu_bank_queries, tokenomics_orange_dao_queries, tokenomics_queries

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

addresess_erc20_721 = {
    "0x64060aB139Feaae7f06Ca4E63189D86aDEb51691": "ERC20",  # UNIM
    "0x431CD3C9AC9Fc73644BF68bF5691f4B83F9E104f": "ERC20",  # RBW
    "0xdC0479CC5BbA033B3e7De9F178607150B3AbCe1f": "NFT",  # unicorns
    "0xA2a13cE1824F3916fC84C65e559391fc6674e6e8": "NFT",  # lands
    "0xa7D50EE3D7485288107664cf758E877a0D351725": "NFT",  # shadowcorns
}

addresess_erc1155 = ["0x99A558BDBdE247C2B2716f0D4cFb0E246DFB697D"]


def generate_report(
    client: Moonstream,
    token: Union[str, UUID],
    query_name: str,
    params: Dict[str, Any],
    bucket_prefix: str,
    bucket: str,
    key: str,
):
    """
    Generate the report.
    """

    try:
        json_data = recive_S3_data_from_query(
            client=client,
            token=token,
            query_name=query_name,
            params=params,
        )

        client.upload_query_results(
            json.dumps(json_data),
            bucket,
            f"{bucket_prefix}/{key}",
        )
        logger.info(
            f"Report generated and results uploaded at: https://{bucket}/{bucket_prefix}/{key}"
        )
    except Exception as err:
        logger.error(
            f"Cant recive or load data for s3, for query: {query_name}, bucket: {bucket}, key: {key}. End with error: {err}"
        )


def create_user_query(
    client: Moonstream,
    token: Union[str, UUID],
    query_name: str,
    query: str,
):
    """
    Create a user query.
    """

    try:
        client.create_query(token=token, name=query_name, query=query)
    except Exception as err:
        logger.error(f"Cant create user query: {query_name}. End with error: {err}")


def delete_user_query(client: Moonstream, token: str, query_name: str):
    """
    Delete the user's queries.
    """

    id = client.delete_query(
        token=token,
        name=query_name,
    )

    logger.info(f"Query with name:{query_name} and id: {id} was deleted")


def init_game_bank_queries_handler(args: argparse.Namespace):
    """
    Create the game bank queries.
    """

    client = Moonstream()

    for query in cu_bank_queries:
        try:
            if args.overwrite:
                try:
                    # delete
                    delete_user_query(
                        client=client,
                        token=args.moonstream_token,
                        query_name=query["name"],
                    )
                except Exception as err:
                    logger.error(err)
            # create
            created_entry = client.create_query(
                token=args.moonstream_token,
                name=query["name"],
                query=query["query"],
            )
            logger.info(
                f"Created query {query['name']} please validate it in the UI url {created_entry.journal_url}/entries/{created_entry.id}/"
            )
        except Exception as e:
            logger.error(e)
            pass


def init_tokenomics_queries_handler(args: argparse.Namespace):
    """
    Create the tokenomics queries.
    """

    client = Moonstream()

    if args.project == "cu":
        queries = tokenomics_queries
    elif args.project == "orangedao":
        queries = tokenomics_orange_dao_queries
    else:
        logger.error("Project not found")
        raise Exception("Project not found")

    for query in queries:
        try:
            if args.overwrite:
                try:
                    # delete
                    delete_user_query(
                        client=client,
                        token=args.moonstream_token,
                        query_name=query["name"],
                    )
                except Exception as err:
                    logger.error(err)
            # create
            created_entry = client.create_query(
                token=args.moonstream_token,
                name=query["name"],
                query=query["query"],
            )
            logger.info(
                f"Created query {query['name']} please validate it in the UI url {created_entry.journal_url}/entries/{created_entry.id}/"
            )
        except Exception as e:
            logger.error(e)
            pass


def run_tokenomics_queries_handler(args: argparse.Namespace):
    client = Moonstream()

    query_name = "erc20_721_volume"

    ### Run voluem query

    ranges = [
        {"time_format": "YYYY-MM-DD HH24", "time_range": "24 hours"},
        {"time_format": "YYYY-MM-DD HH24", "time_range": "7 days"},
        {"time_format": "YYYY-MM-DD", "time_range": "30 days"},
    ]

    # volume of erc20 and erc721

    for address, type in addresess_erc20_721.items():
        for range in ranges:
            params: Dict[str, Any] = {
                "address": address,
                "type": type,
                "time_format": range["time_format"],
                "time_range": range["time_range"],
            }

            generate_report(
                client=client,
                token=args.moonstream_token,
                query_name=query_name,
                params=params,
                bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
                bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
                key=f'{query_name}/{address}/{range["time_range"].replace(" ","_")}/data.json',
            )

    # volume change of erc20 and erc721

    query_name = "volume_change"

    for address, type in addresess_erc20_721.items():
        for range in ranges:
            params = {
                "address": address,
                "type": type,
                "time_range": range["time_range"],
            }

            generate_report(
                client=client,
                token=args.moonstream_token,
                query_name=query_name,
                params=params,
                bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
                bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
                key=f'{query_name}/{address}/{range["time_range"].replace(" ","_")}/data.json',
            )

    query_name = "erc1155_volume"

    # volume of erc1155

    addresess_erc1155 = ["0x99A558BDBdE247C2B2716f0D4cFb0E246DFB697D"]

    for address in addresess_erc1155:
        for range in ranges:
            params = {
                "address": address,
                "time_format": range["time_format"],
                "time_range": range["time_range"],
            }

            generate_report(
                client=client,
                token=args.moonstream_token,
                query_name=query_name,
                params=params,
                bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
                bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
                key=f"{query_name}/{address}/{range['time_range'].replace(' ','_')}/data.json",
            )

    # most_recent_sale

    query_name = "most_recent_sale"

    for address, type in addresess_erc20_721.items():
        if type == "NFT":
            for amount in [10, 100]:
                params = {
                    "address": address,
                    "amount": amount,
                }

                generate_report(
                    client=client,
                    token=args.moonstream_token,
                    query_name=query_name,
                    params=params,
                    bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
                    bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
                    key=f"{query_name}/{address}/{amount}/data.json",
                )

    # most_active_buyers

    query_name = "most_active_buyers"

    for address, type in addresess_erc20_721.items():
        if type == "NFT":
            for range in ranges:
                params = {
                    "address": address,
                    "time_range": range["time_range"],
                }

                generate_report(
                    client=client,
                    token=args.moonstream_token,
                    query_name=query_name,
                    params=params,
                    bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
                    bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
                    key=f"{query_name}/{address}/{range['time_range'].replace(' ','_')}/data.json",
                )

    # most_active_sellers

    query_name = "most_active_sellers"

    for address, type in addresess_erc20_721.items():
        if type == "NFT":
            for range in ranges:
                params = {
                    "address": address,
                    "time_range": range["time_range"],
                }

                generate_report(
                    client=client,
                    token=args.moonstream_token,
                    query_name=query_name,
                    params=params,
                    bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
                    bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
                    key=f"{query_name}/{address}/{range['time_range'].replace(' ','_')}/data.json",
                )

    # lagerst_owners

    query_name = "lagerst_owners"
    for address, type in addresess_erc20_721.items():
        if type == "NFT":
            params = {
                "address": address,
            }

            generate_report(
                client=client,
                token=args.moonstream_token,
                query_name=query_name,
                params=params,
                bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
                bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
                key=f"{query_name}/{address}/data.json",
            )

    # total_supply_erc721

    query_name = "total_supply_erc721"

    for address, type in addresess_erc20_721.items():
        if type == "NFT":
            params = {
                "address": address,
            }

            generate_report(
                client=client,
                token=args.moonstream_token,
                query_name=query_name,
                params=params,
                bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
                bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
                key=f"{query_name}/{address}/data.json",
            )

    # total_supply_terminus

    query_name = "total_supply_terminus"

    for address in addresess_erc1155:
        params = {
            "address": address,
        }

        generate_report(
            client=client,
            token=args.moonstream_token,
            query_name=query_name,
            params=params,
            bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
            bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
            key=f"{query_name}/{address}/data.json",
        )

    logger.info("Done")


def list_user_queries_handler(args: argparse.Namespace):
    """
    List the user's queries.
    """

    client = Moonstream()

    queries = client.list_queries(
        token=args.moonstream_token,
    )

    for query in queries.queries:
        logger.info(query.name, query.id)


def delete_user_query_handler(args: argparse.Namespace):
    """
    Delete the user's queries.
    """
    client = Moonstream()

    delete_user_query(client=client, token=args.moonstream_token, query_name=args.name)


def create_user_query_handler(args: argparse.Namespace):
    """
    Create the user's queries.
    """
    client = Moonstream()

    for query in tokenomics_queries:
        if query["name"] == args.name:
            create_user_query(
                client=client,
                token=args.moonstream_token,
                query_name=query["name"],
                query=query["query"],
            )


def generate_game_bank_report(args: argparse.Namespace):
    """
    han
    Generate the game bank query.
    """

    client = Moonstream()

    for query in client.list_queries(
        token=args.moonstream_token,
    ).queries:
        params = {}

        if (
            query.name == "cu-bank-withdrawals-total"
            or query.name == "cu-bank-withdrawals-events"
        ):
            blocktimestamp = int(time.time())
            params = {"block_timestamp": blocktimestamp}

        keep_going = True

        if_modified_since_datetime = datetime.datetime.utcnow()
        if_modified_since = if_modified_since_datetime.strftime(
            "%a, %d %b %Y %H:%M:%S GMT"
        )

        data_url = client.exec_query(
            token=args.moonstream_token,
            query_name=query.name,
            params=params,
        )  # S3 presign_url

        while keep_going:
            data_response = requests.get(
                data_url,
                headers={"If-Modified-Since": if_modified_since},
                timeout=10,
            )
            # push to s3

            if data_response.status_code == 200:
                json.dumps(data_response.json())
                break
            else:
                time.sleep(2)
                continue

    pass


def generate_report_nft_dashboard_handler(args: argparse.Namespace):
    """
    Generate report from metadata crawler and push to s3
    use query API
    """

    client = Moonstream()

    for query in client.list_queries(
        token=args.moonstream_token,
    ).queries:
        params = {}  # type: ignore

        if query.name not in [
            "cu_nft_dashboard_data",
            "cu_land_nft_dashboard_data",
            "cu_seaport_feed_polygon",
            "cu_breeding_feed",
        ]:
            continue

        logger.info(f"Generating report for {query.name}")
        data = recive_S3_data_from_query(
            client=client,
            token=args.moonstream_token,
            query_name=query.name,
            params=params,
            time_await=4,
        )

        logger.info(f"Data recived. Uploading report for {query.name} as json")

        # send as json
        ext = "json"

        url = client.upload_query_results(
            json.dumps(data),
            key=f"queries/{query.name}/data.{ext}",
            bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
        )

        logger.info(f"Report uploaded to {url}")

        logger.info(f"Data recived. Uploading report for {query.name} as csv")

        ext = "csv"
        csv_buffer = StringIO()

        dict_csv_writer = csv.DictWriter(
            csv_buffer, fieldnames=data["data"][0].keys(), delimiter=","
        )

        # upload to s3 bucket as csv
        dict_csv_writer.writeheader()
        dict_csv_writer.writerows(data["data"])

        url = client.upload_query_results(
            data=csv_buffer.getvalue().encode("utf-8"),
            key=f"queries/{query.name}/data.{ext}",
            bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
        )

        logger.info(f"Report uploaded to {url}")


def run_tokenomics_orange_dao_handler(args: argparse.Namespace):
    client = Moonstream()

    params: Dict[str, Any] = {}

    ranges = [
        {"time_format": "YYYY-MM-DD HH24", "time_range": "24 hours"},
        {"time_format": "YYYY-MM-DD HH24", "time_range": "7 days"},
        {"time_format": "YYYY-MM-DD", "time_range": "30 days"},
    ]

    # volume of erc20 and erc721

    query_name = "balances_by_address"

    generate_report(
        client=client,
        token=args.moonstream_token,
        query_name=query_name,
        params=params,
        bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
        bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
        key=f"orangedao/{query_name}/data.json",
    )

    query_name = "transfers_feed"

    for range in ranges:
        params = {
            "time_range": range["time_range"],
        }

        generate_report(
            client=client,
            token=args.moonstream_token,
            query_name=query_name,
            params=params,
            bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
            bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
            key=f"orangedao/{query_name}/{range['time_range'].replace(' ','_')}_data.json",
        )

    # all time feed

    params = {
        "time_range": f"{int(time.time())} seconds",
    }

    generate_report(
        client=client,
        token=args.moonstream_token,
        query_name=query_name,
        params=params,
        bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
        bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
        key=f"orangedao/{query_name}/all_time_data.json",
    )

    limits = [10, 100]

    query_name = "largest_distributors"

    for limit in limits:
        params = {
            "limit": limit,
        }

        generate_report(
            client=client,
            token=args.moonstream_token,
            query_name=query_name,
            params=params,
            bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
            bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
            key=f"orangedao/{query_name}/{limit}_data.json",
        )

    query_name = "largest_recipients"

    for limit in limits:
        params = {
            "limit": limit,
        }

        generate_report(
            client=client,
            token=args.moonstream_token,
            query_name=query_name,
            params=params,
            bucket_prefix=MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
            bucket=MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
            key=f"orangedao/{query_name}/{limit}_data.json",
        )

    logger.info("Done")


def main():
    parser = argparse.ArgumentParser()

    parser.set_defaults(func=lambda _: parser.print_help())

    subparsers = parser.add_subparsers()

    general_queries_parser = subparsers.add_parser("queries", help="Queries commands")

    general_queries_subparsers = general_queries_parser.add_subparsers()

    general_queries_parser.add_argument(
        "--moonstream-token",
        required=True,
        type=str,
    )

    general_queries_subparsers.add_parser(
        "list",
        help="List all queries",
        description="List all queries",
    ).set_defaults(func=list_user_queries_handler)

    init_tokenonomics_parser = general_queries_subparsers.add_parser(
        "init-tokenonomics",
        help="Create all predifind query",
        description="Create all predifind query",
    )

    init_tokenonomics_parser.add_argument(
        "--project",
        choices=["cu", "orangedao"],
        required=True,
        type=str,
    )

    init_tokenonomics_parser.add_argument("--overwrite", type=bool, default=False)

    init_tokenonomics_parser.set_defaults(func=init_tokenomics_queries_handler)

    cu_reports_parser = subparsers.add_parser("cu-reports", help="CU Reports")

    cu_reports_subparsers = cu_reports_parser.add_subparsers()

    cu_reports_parser.add_argument(
        "--moonstream-token",
        required=True,
        type=str,
    )
    queries_parser = cu_reports_subparsers.add_parser(
        "queries",
        help="Queries commands",
    )

    queries_parser.set_defaults(func=lambda _: queries_parser.print_help())

    queries_subparsers = queries_parser.add_subparsers()

    queries_subparsers.add_parser(
        "list",
        help="List all queries",
        description="List all queries",
    ).set_defaults(func=list_user_queries_handler)

    init_game_bank_parser = queries_subparsers.add_parser(
        "init-game-bank",
        help="Create all predifind query",
        description="Create all predifind query",
    )

    init_game_bank_parser.add_argument("--overwrite", type=bool, default=False)

    init_game_bank_parser.set_defaults(func=init_game_bank_queries_handler)

    generate_report = queries_subparsers.add_parser(
        "run-tokenonomics",
        help="Run tokenomics queries and push to S3 public backet",
        description="Run tokenomics queries and push to S3 public backet",
    )

    generate_report.set_defaults(func=run_tokenomics_queries_handler)

    delete_query = queries_subparsers.add_parser(
        "delete",
        help="Delete all predifind query",
        description="Delete all predifind query",
    )

    delete_query.add_argument(
        "--name",
        required=True,
        type=str,
    )

    delete_query.set_defaults(func=delete_user_query_handler)

    create_query = queries_subparsers.add_parser(
        "create",
        help="Create all predifind query",
        description="Create all predifind query",
    )

    create_query.add_argument(
        "--name",
        required=True,
        type=str,
    )

    create_query.set_defaults(func=create_user_query_handler)

    cu_bank_parser = cu_reports_subparsers.add_parser(
        "generate-reports",
        help="Generate cu-bank state reports",
    )

    cu_bank_parser.set_defaults(func=generate_game_bank_report)

    cu_nft_dashboard_parser = cu_reports_subparsers.add_parser(
        "generate-nft-dashboard",
        help="Generate cu-nft-dashboard",
    )
    cu_nft_dashboard_parser.set_defaults(func=generate_report_nft_dashboard_handler)

    orangedao_parser = subparsers.add_parser(
        "orangedao",
        help="Orange DAO commands",
    )

    orangedao_parser.add_argument(
        "--moonstream-token",
        required=True,
        type=str,
    )

    orangedao_parser.set_defaults(func=lambda _: orangedao_parser.print_help())

    orangedao_subparsers = orangedao_parser.add_subparsers()

    orangedao_subparsers.add_parser(
        "run_tokenomics_orange_dao",
        help="Generate Orange DAO reports",
    ).set_defaults(func=run_tokenomics_orange_dao_handler)

    args = parser.parse_args()
    args.func(args)


if __name__ == "__main__":
    main()