kopia lustrzana https://github.com/bugout-dev/moonstream
314 wiersze
9.8 KiB
Python
314 wiersze
9.8 KiB
Python
"""
|
|
The Mooncrawl HTTP API
|
|
"""
|
|
|
|
import logging
|
|
import time
|
|
from cgi import test
|
|
from datetime import timedelta
|
|
from typing import Any, Dict, List
|
|
from uuid import UUID
|
|
|
|
import boto3 # type: ignore
|
|
from bugout.data import BugoutJournalEntity, BugoutResource
|
|
from fastapi import BackgroundTasks, FastAPI
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from moonstreamdb.blockchain import (
|
|
AvailableBlockchainType,
|
|
get_block_model,
|
|
get_label_model,
|
|
get_transaction_model,
|
|
)
|
|
from sqlalchemy import text
|
|
|
|
from . import data
|
|
from .actions import (
|
|
EntityCollectionNotFoundException,
|
|
generate_s3_access_links,
|
|
get_entity_subscription_collection_id,
|
|
query_parameter_hash,
|
|
)
|
|
from .middleware import MoonstreamHTTPException
|
|
from .settings import (
|
|
BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION,
|
|
DOCS_TARGET_PATH,
|
|
LINKS_EXPIRATION_TIME,
|
|
MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
|
MOONSTREAM_S3_QUERIES_BUCKET,
|
|
MOONSTREAM_S3_QUERIES_BUCKET_PREFIX,
|
|
MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
|
|
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
|
|
ORIGINS,
|
|
)
|
|
from .settings import bugout_client as bc
|
|
from .stats_worker import dashboard, queries
|
|
from .version import MOONCRAWL_VERSION
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
tags_metadata = [
|
|
{"name": "jobs", "description": "Trigger crawler jobs."},
|
|
{"name": "time", "description": "Server timestamp endpoints."},
|
|
]
|
|
|
|
app = FastAPI(
|
|
title=f"Mooncrawl HTTP API",
|
|
description="Mooncrawl API endpoints.",
|
|
version=MOONCRAWL_VERSION,
|
|
openapi_tags=tags_metadata,
|
|
openapi_url="/openapi.json",
|
|
docs_url=None,
|
|
redoc_url=f"/{DOCS_TARGET_PATH}",
|
|
)
|
|
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=ORIGINS,
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
|
|
@app.get("/ping", response_model=data.PingResponse)
|
|
async def ping_handler() -> data.PingResponse:
|
|
"""
|
|
Check server status.
|
|
"""
|
|
return data.PingResponse(status="ok")
|
|
|
|
|
|
@app.get("/version", response_model=data.VersionResponse)
|
|
async def version_handler() -> data.VersionResponse:
|
|
"""
|
|
Get server version.
|
|
"""
|
|
return data.VersionResponse(version=MOONCRAWL_VERSION)
|
|
|
|
|
|
@app.get("/now", tags=["time"])
|
|
async def now_handler() -> data.NowResponse:
|
|
"""
|
|
Get server current time.
|
|
"""
|
|
return data.NowResponse(epoch_time=time.time())
|
|
|
|
|
|
@app.post("/jobs/stats_update", tags=["jobs"])
|
|
async def status_handler(
|
|
stats_update: data.StatsUpdateRequest,
|
|
background_tasks: BackgroundTasks,
|
|
):
|
|
"""
|
|
Update dashboard endpoint create are tasks for update.
|
|
"""
|
|
|
|
dashboard_resource: BugoutResource = bc.get_resource(
|
|
token=stats_update.token,
|
|
resource_id=stats_update.dashboard_id,
|
|
timeout=10,
|
|
)
|
|
|
|
try:
|
|
journal_id = get_entity_subscription_collection_id(
|
|
resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION,
|
|
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
|
user_id=UUID(stats_update.user_id),
|
|
)
|
|
except EntityCollectionNotFoundException as e:
|
|
raise MoonstreamHTTPException(
|
|
status_code=404,
|
|
detail="User subscriptions collection not found",
|
|
internal_error=e,
|
|
)
|
|
except Exception as e:
|
|
logger.error(
|
|
f"Error listing subscriptions for user ({stats_update.user_id}) with token: {stats_update.token}, error: {str(e)}"
|
|
)
|
|
|
|
# get subscription entities
|
|
|
|
s3_client = boto3.client("s3")
|
|
|
|
subscription_by_id: Dict[str, BugoutJournalEntity] = {}
|
|
|
|
for dashboard_subscription_filters in dashboard_resource.resource_data[
|
|
"subscription_settings"
|
|
]:
|
|
# get subscription by id
|
|
subscription: BugoutJournalEntity = bc.get_entity(
|
|
token=stats_update.token,
|
|
journal_id=journal_id,
|
|
entity_id=dashboard_subscription_filters["subscription_id"],
|
|
)
|
|
|
|
subscription_by_id[str(subscription.id)] = subscription
|
|
|
|
try:
|
|
background_tasks.add_task(
|
|
dashboard.stats_generate_api_task,
|
|
timescales=stats_update.timescales,
|
|
dashboard=dashboard_resource,
|
|
subscription_by_id=subscription_by_id,
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(
|
|
f"Unhandled /jobs/stats_update start background task exception, error: {e}"
|
|
)
|
|
raise MoonstreamHTTPException(status_code=500)
|
|
|
|
presigned_urls_response: Dict[UUID, Any] = {}
|
|
|
|
for dashboard_subscription_filters in dashboard_resource.resource_data[
|
|
"subscription_settings"
|
|
]:
|
|
# get subscription by id
|
|
|
|
subscription_entity = subscription_by_id[
|
|
dashboard_subscription_filters["subscription_id"]
|
|
]
|
|
|
|
for reqired_field in subscription.required_fields: # type: ignore
|
|
if "subscription_type_id" in reqired_field:
|
|
subscriprions_type = reqired_field["subscription_type_id"]
|
|
|
|
for timescale in stats_update.timescales:
|
|
presigned_urls_response[subscription_entity.id] = {}
|
|
|
|
try:
|
|
result_key = f"{MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX}/{dashboard.blockchain_by_subscription_id[subscriprions_type]}/contracts_data/{subscription_entity.address}/{stats_update.dashboard_id}/v1/{timescale}.json"
|
|
|
|
object = s3_client.head_object(
|
|
Bucket=MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, Key=result_key
|
|
)
|
|
|
|
stats_presigned_url = s3_client.generate_presigned_url(
|
|
"get_object",
|
|
Params={
|
|
"Bucket": MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
|
|
"Key": result_key,
|
|
},
|
|
ExpiresIn=300,
|
|
HttpMethod="GET",
|
|
)
|
|
|
|
presigned_urls_response[subscription_entity.id][timescale] = {
|
|
"url": stats_presigned_url,
|
|
"headers": {
|
|
"If-Modified-Since": (
|
|
object["LastModified"] + timedelta(seconds=1)
|
|
).strftime("%c")
|
|
},
|
|
}
|
|
except Exception as err:
|
|
logger.warning(
|
|
f"Can't generate S3 presigned url in stats endpoint for Bucket:{MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET}, Key:{result_key} get error:{err}"
|
|
)
|
|
|
|
return presigned_urls_response
|
|
|
|
|
|
@app.post("/jobs/{query_id}/query_update", tags=["jobs"])
|
|
async def queries_data_update_handler(
|
|
query_id: str,
|
|
request_data: data.QueryDataUpdate,
|
|
background_tasks: BackgroundTasks,
|
|
) -> Dict[str, Any]:
|
|
# Check if query is valid
|
|
try:
|
|
queries.query_validation(request_data.query)
|
|
except queries.QueryNotValid:
|
|
logger.error(f"Query not pass validation check query id: {query_id}")
|
|
raise MoonstreamHTTPException(
|
|
status_code=401,
|
|
detail="Incorrect query is not valid with current restrictions",
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Unhandled query execute exception, error: {e}")
|
|
raise MoonstreamHTTPException(status_code=500)
|
|
|
|
requested_query = request_data.query
|
|
|
|
if request_data.blockchain:
|
|
if request_data.blockchain not in [i.value for i in AvailableBlockchainType]:
|
|
logger.error(f"Unknown blockchain {request_data.blockchain}")
|
|
raise MoonstreamHTTPException(status_code=403, detail="Unknown blockchain")
|
|
|
|
blockchain = AvailableBlockchainType(request_data.blockchain)
|
|
|
|
requested_query = (
|
|
requested_query.replace(
|
|
"__transactions_table__",
|
|
get_transaction_model(blockchain).__tablename__,
|
|
)
|
|
.replace(
|
|
"__blocks_table__",
|
|
get_block_model(blockchain).__tablename__,
|
|
)
|
|
.replace(
|
|
"__labels_table__",
|
|
get_label_model(blockchain).__tablename__,
|
|
)
|
|
)
|
|
|
|
# Check if it can transform to TextClause
|
|
try:
|
|
query = text(requested_query)
|
|
except Exception as e:
|
|
logger.error(
|
|
f"Can't parse query {query_id} to TextClause in drones /query_update endpoint, error: {e}"
|
|
)
|
|
raise MoonstreamHTTPException(status_code=500, detail="Can't parse query")
|
|
|
|
# Get requried keys for query
|
|
expected_query_parameters = query._bindparams.keys()
|
|
|
|
# request.params validations
|
|
passed_params = {
|
|
key: queries.from_json_types(value)
|
|
for key, value in request_data.params.items()
|
|
if key in expected_query_parameters
|
|
}
|
|
|
|
if len(passed_params) != len(expected_query_parameters):
|
|
logger.error(
|
|
f"Unmatched amount of applying query parameters: {passed_params}, query_id:{query_id}."
|
|
)
|
|
raise MoonstreamHTTPException(
|
|
status_code=500, detail="Unmatched amount of applying query parameters"
|
|
)
|
|
|
|
params_hash = query_parameter_hash(passed_params)
|
|
|
|
bucket = MOONSTREAM_S3_QUERIES_BUCKET
|
|
key = f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/{params_hash}/data.{request_data.file_type}"
|
|
|
|
try:
|
|
background_tasks.add_task(
|
|
queries.data_generate,
|
|
query_id=f"{query_id}",
|
|
file_type=request_data.file_type,
|
|
bucket=bucket,
|
|
key=key,
|
|
query=query,
|
|
params=passed_params,
|
|
params_hash=params_hash,
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Unhandled query execute exception, error: {e}")
|
|
raise MoonstreamHTTPException(status_code=500)
|
|
|
|
stats_presigned_url = generate_s3_access_links(
|
|
method_name="get_object",
|
|
bucket=bucket,
|
|
key=key,
|
|
expiration=LINKS_EXPIRATION_TIME,
|
|
http_method="GET",
|
|
)
|
|
|
|
return {"url": stats_presigned_url}
|