moonstream/crawlers/mooncrawl/mooncrawl/api.py

"""
The Mooncrawl HTTP API
"""

import logging
import time
from cgi import test
from datetime import timedelta
from typing import Any, Dict, List
from uuid import UUID

import boto3  # type: ignore
from bugout.data import BugoutJournalEntity, BugoutResource
from fastapi import BackgroundTasks, FastAPI
from fastapi.middleware.cors import CORSMiddleware
from moonstreamdb.blockchain import (
    AvailableBlockchainType,
    get_block_model,
    get_label_model,
    get_transaction_model,
)
from sqlalchemy import text

from . import data
from .actions import (
    EntityCollectionNotFoundException,
    generate_s3_access_links,
    get_entity_subscription_collection_id,
    query_parameter_hash,
)
from .middleware import MoonstreamHTTPException
from .settings import (
    BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION,
    DOCS_TARGET_PATH,
    LINKS_EXPIRATION_TIME,
    MOONSTREAM_ADMIN_ACCESS_TOKEN,
    MOONSTREAM_S3_QUERIES_BUCKET,
    MOONSTREAM_S3_QUERIES_BUCKET_PREFIX,
    MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
    MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
    ORIGINS,
)
from .settings import bugout_client as bc
from .stats_worker import dashboard, queries
from .version import MOONCRAWL_VERSION

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


tags_metadata = [
    {"name": "jobs", "description": "Trigger crawler jobs."},
    {"name": "time", "description": "Server timestamp endpoints."},
]

app = FastAPI(
    title=f"Mooncrawl HTTP API",
    description="Mooncrawl API endpoints.",
    version=MOONCRAWL_VERSION,
    openapi_tags=tags_metadata,
    openapi_url="/openapi.json",
    docs_url=None,
    redoc_url=f"/{DOCS_TARGET_PATH}",
)

app.add_middleware(
    CORSMiddleware,
    allow_origins=ORIGINS,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)


@app.get("/ping", response_model=data.PingResponse)
async def ping_handler() -> data.PingResponse:
    """
    Check server status.
    """
    return data.PingResponse(status="ok")


@app.get("/version", response_model=data.VersionResponse)
async def version_handler() -> data.VersionResponse:
    """
    Get server version.
    """
    return data.VersionResponse(version=MOONCRAWL_VERSION)


@app.get("/now", tags=["time"])
async def now_handler() -> data.NowResponse:
    """
    Get server current time.
    """
    return data.NowResponse(epoch_time=time.time())


@app.post("/jobs/stats_update", tags=["jobs"])
async def status_handler(
    stats_update: data.StatsUpdateRequest,
    background_tasks: BackgroundTasks,
):
    """
    Update dashboard endpoint create are tasks for update.
    """

    dashboard_resource: BugoutResource = bc.get_resource(
        token=stats_update.token,
        resource_id=stats_update.dashboard_id,
        timeout=10,
    )

    try:
        journal_id = get_entity_subscription_collection_id(
            resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION,
            token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
            user_id=UUID(stats_update.user_id),
        )
    except EntityCollectionNotFoundException as e:
        raise MoonstreamHTTPException(
            status_code=404,
            detail="User subscriptions collection not found",
            internal_error=e,
        )
    except Exception as e:
        logger.error(
            f"Error listing subscriptions for user ({stats_update.user_id}) with token: {stats_update.token}, error: {str(e)}"
        )

    # get subscription entities

    s3_client = boto3.client("s3")

    subscription_by_id: Dict[str, BugoutJournalEntity] = {}

    for dashboard_subscription_filters in dashboard_resource.resource_data[
        "subscription_settings"
    ]:
        # get subscription by id
        subscription: BugoutJournalEntity = bc.get_entity(
            token=stats_update.token,
            journal_id=journal_id,
            entity_id=dashboard_subscription_filters["subscription_id"],
        )

        subscription_by_id[str(subscription.id)] = subscription

    try:
        background_tasks.add_task(
            dashboard.stats_generate_api_task,
            timescales=stats_update.timescales,
            dashboard=dashboard_resource,
            subscription_by_id=subscription_by_id,
        )

    except Exception as e:
        logger.error(
            f"Unhandled /jobs/stats_update start background task exception, error: {e}"
        )
        raise MoonstreamHTTPException(status_code=500)

    presigned_urls_response: Dict[UUID, Any] = {}

    for dashboard_subscription_filters in dashboard_resource.resource_data[
        "subscription_settings"
    ]:
        # get subscription by id

        subscription_entity = subscription_by_id[
            dashboard_subscription_filters["subscription_id"]
        ]

        for reqired_field in subscription.required_fields:  # type: ignore
            if "subscription_type_id" in reqired_field:
                subscriprions_type = reqired_field["subscription_type_id"]

        for timescale in stats_update.timescales:
            presigned_urls_response[subscription_entity.id] = {}

            try:
                result_key = f"{MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX}/{dashboard.blockchain_by_subscription_id[subscriprions_type]}/contracts_data/{subscription_entity.address}/{stats_update.dashboard_id}/v1/{timescale}.json"

                object = s3_client.head_object(
                    Bucket=MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, Key=result_key
                )

                stats_presigned_url = s3_client.generate_presigned_url(
                    "get_object",
                    Params={
                        "Bucket": MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
                        "Key": result_key,
                    },
                    ExpiresIn=300,
                    HttpMethod="GET",
                )

                presigned_urls_response[subscription_entity.id][timescale] = {
                    "url": stats_presigned_url,
                    "headers": {
                        "If-Modified-Since": (
                            object["LastModified"] + timedelta(seconds=1)
                        ).strftime("%c")
                    },
                }
            except Exception as err:
                logger.warning(
                    f"Can't generate S3 presigned url in stats endpoint for Bucket:{MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET}, Key:{result_key} get error:{err}"
                )

    return presigned_urls_response


@app.post("/jobs/{query_id}/query_update", tags=["jobs"])
async def queries_data_update_handler(
    query_id: str,
    request_data: data.QueryDataUpdate,
    background_tasks: BackgroundTasks,
) -> Dict[str, Any]:
    # Check if query is valid
    try:
        queries.query_validation(request_data.query)
    except queries.QueryNotValid:
        logger.error(f"Query not pass validation check query id: {query_id}")
        raise MoonstreamHTTPException(
            status_code=401,
            detail="Incorrect query is not valid with current restrictions",
        )
    except Exception as e:
        logger.error(f"Unhandled query execute exception, error: {e}")
        raise MoonstreamHTTPException(status_code=500)

    requested_query = request_data.query

    if request_data.blockchain:
        if request_data.blockchain not in [i.value for i in AvailableBlockchainType]:
            logger.error(f"Unknown blockchain {request_data.blockchain}")
            raise MoonstreamHTTPException(status_code=403, detail="Unknown blockchain")

        blockchain = AvailableBlockchainType(request_data.blockchain)

        requested_query = (
            requested_query.replace(
                "__transactions_table__",
                get_transaction_model(blockchain).__tablename__,
            )
            .replace(
                "__blocks_table__",
                get_block_model(blockchain).__tablename__,
            )
            .replace(
                "__labels_table__",
                get_label_model(blockchain).__tablename__,
            )
        )

    # Check if it can transform to TextClause
    try:
        query = text(requested_query)
    except Exception as e:
        logger.error(
            f"Can't parse query {query_id} to TextClause in drones /query_update endpoint, error: {e}"
        )
        raise MoonstreamHTTPException(status_code=500, detail="Can't parse query")

    # Get requried keys for query
    expected_query_parameters = query._bindparams.keys()

    # request.params validations
    passed_params = {
        key: queries.from_json_types(value)
        for key, value in request_data.params.items()
        if key in expected_query_parameters
    }

    if len(passed_params) != len(expected_query_parameters):
        logger.error(
            f"Unmatched amount of applying query parameters: {passed_params}, query_id:{query_id}."
        )
        raise MoonstreamHTTPException(
            status_code=500, detail="Unmatched amount of applying query parameters"
        )

    params_hash = query_parameter_hash(passed_params)

    bucket = MOONSTREAM_S3_QUERIES_BUCKET
    key = f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/{params_hash}/data.{request_data.file_type}"

    try:
        background_tasks.add_task(
            queries.data_generate,
            query_id=f"{query_id}",
            file_type=request_data.file_type,
            bucket=bucket,
            key=key,
            query=query,
            params=passed_params,
            params_hash=params_hash,
        )

    except Exception as e:
        logger.error(f"Unhandled query execute exception, error: {e}")
        raise MoonstreamHTTPException(status_code=500)

    stats_presigned_url = generate_s3_access_links(
        method_name="get_object",
        bucket=bucket,
        key=key,
        expiration=LINKS_EXPIRATION_TIME,
        http_method="GET",
    )

    return {"url": stats_presigned_url}