kopia lustrzana https://github.com/bugout-dev/moonstream
API endpoint for crawlers
rodzic
42e764d1b2
commit
11aa45690f
|
@ -0,0 +1,81 @@
|
|||
"""
|
||||
The Mooncrawl HTTP API
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from typing import Dict
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from . import data
|
||||
from .middleware import MoonstreamHTTPException
|
||||
from .settings import DOCS_TARGET_PATH, ORIGINS
|
||||
from .version import MOONCRAWL_VERSION
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
tags_metadata = [
|
||||
{"name": "jobs", "description": "Trigger crawler jobs."},
|
||||
{"name": "time", "description": "Server timestamp endpoints."},
|
||||
]
|
||||
|
||||
app = FastAPI(
|
||||
title=f"Mooncrawl HTTP API",
|
||||
description="Mooncrawl API endpoints.",
|
||||
version=MOONCRAWL_VERSION,
|
||||
openapi_tags=tags_metadata,
|
||||
openapi_url="/openapi.json",
|
||||
docs_url=None,
|
||||
redoc_url=f"/{DOCS_TARGET_PATH}",
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=ORIGINS,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.get("/ping", response_model=data.PingResponse)
|
||||
async def ping_handler() -> data.PingResponse:
|
||||
"""
|
||||
Check server status.
|
||||
"""
|
||||
return data.PingResponse(status="ok")
|
||||
|
||||
|
||||
@app.get("/version", response_model=data.VersionResponse)
|
||||
async def version_handler() -> data.VersionResponse:
|
||||
"""
|
||||
Get server version.
|
||||
"""
|
||||
return data.VersionResponse(version=MOONCRAWL_VERSION)
|
||||
|
||||
|
||||
@app.get("/now", tags=["time"])
|
||||
async def now_handler() -> data.NowResponse:
|
||||
"""
|
||||
Get server current time.
|
||||
"""
|
||||
return data.NowResponse(epoch_time=time.time())
|
||||
|
||||
|
||||
@app.get("/jobs/stats_update", tags=["jobs"])
|
||||
async def status_handler():
|
||||
"""
|
||||
Find latest crawlers records with creation timestamp:
|
||||
- ethereum_txpool
|
||||
- ethereum_trending
|
||||
"""
|
||||
try:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"Unhandled status exception, error: {e}")
|
||||
raise MoonstreamHTTPException(status_code=500)
|
||||
|
||||
return
|
|
@ -2,6 +2,8 @@ from dataclasses import dataclass
|
|||
from datetime import datetime
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class AvailableBlockchainType(Enum):
|
||||
ETHEREUM = "ethereum"
|
||||
|
@ -14,3 +16,27 @@ class DateRange:
|
|||
end_time: datetime
|
||||
include_start: bool
|
||||
include_end: bool
|
||||
|
||||
|
||||
class PingResponse(BaseModel):
|
||||
"""
|
||||
Schema for ping response
|
||||
"""
|
||||
|
||||
status: str
|
||||
|
||||
|
||||
class VersionResponse(BaseModel):
|
||||
"""
|
||||
Schema for responses on /version endpoint
|
||||
"""
|
||||
|
||||
version: str
|
||||
|
||||
|
||||
class NowResponse(BaseModel):
|
||||
"""
|
||||
Schema for responses on /now endpoint
|
||||
"""
|
||||
|
||||
epoch_time: float
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
import logging
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from .reporter import reporter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MoonstreamHTTPException(HTTPException):
|
||||
"""
|
||||
Extended HTTPException to handle 500 Internal server errors
|
||||
and send crash reports.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
detail: Any = None,
|
||||
headers: Optional[Dict[str, Any]] = None,
|
||||
internal_error: Exception = None,
|
||||
):
|
||||
super().__init__(status_code, detail, headers)
|
||||
if internal_error is not None:
|
||||
reporter.error_report(internal_error)
|
|
@ -6,10 +6,21 @@ from bugout.app import Bugout
|
|||
# Bugout
|
||||
BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev")
|
||||
BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev")
|
||||
bc = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL)
|
||||
bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL)
|
||||
|
||||
HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
|
||||
|
||||
# Origin
|
||||
RAW_ORIGINS = os.environ.get("MOONSTREAM_CORS_ALLOWED_ORIGINS")
|
||||
if RAW_ORIGINS is None:
|
||||
raise ValueError(
|
||||
"MOONSTREAM_CORS_ALLOWED_ORIGINS environment variable must be set (comma-separated list of CORS allowed origins)"
|
||||
)
|
||||
ORIGINS = RAW_ORIGINS.split(",")
|
||||
|
||||
# OpenAPI
|
||||
DOCS_TARGET_PATH = "docs"
|
||||
|
||||
# Geth connection address
|
||||
MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.environ.get(
|
||||
"MOONSTREAM_NODE_ETHEREUM_IPC_ADDR", "127.0.0.1"
|
||||
|
@ -56,4 +67,15 @@ MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID", "")
|
|||
if MOONSTREAM_DATA_JOURNAL_ID == "":
|
||||
raise ValueError("MOONSTREAM_DATA_JOURNAL_ID env variable is not set")
|
||||
|
||||
AWS_S3_SMARTCONTRACTS_ABI_PREFIX = os.getenv("AWS_S3_SMARTCONTRACTS_ABI_PREFIX")
|
||||
# AWS S3 bucket for ABI smartcontract settings
|
||||
AWS_S3_SMARTCONTRACTS_ABI_BUCKET = os.environ.get("AWS_S3_SMARTCONTRACTS_ABI_BUCKET")
|
||||
if AWS_S3_SMARTCONTRACTS_ABI_BUCKET is None:
|
||||
raise ValueError(
|
||||
"AWS_S3_SMARTCONTRACTS_ABI_BUCKET environment variable must be set"
|
||||
)
|
||||
AWS_S3_SMARTCONTRACTS_ABI_PREFIX = os.environ.get("AWS_S3_SMARTCONTRACTS_ABI_PREFIX")
|
||||
if AWS_S3_SMARTCONTRACTS_ABI_PREFIX is None:
|
||||
raise ValueError(
|
||||
"AWS_S3_SMARTCONTRACTS_ABI_PREFIX environment variable must be set"
|
||||
)
|
||||
AWS_S3_SMARTCONTRACTS_ABI_PREFIX = AWS_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip("/")
|
||||
|
|
|
@ -19,10 +19,11 @@ from sqlalchemy.orm import Query, Session
|
|||
from ..blockchain import get_block_model, get_label_model, get_transaction_model
|
||||
from ..data import AvailableBlockchainType
|
||||
from ..settings import (
|
||||
AWS_S3_SMARTCONTRACTS_ABI_BUCKET,
|
||||
AWS_S3_SMARTCONTRACTS_ABI_PREFIX,
|
||||
MOONSTREAM_ADMIN_ACCESS_TOKEN,
|
||||
bc,
|
||||
)
|
||||
from ..settings import bugout_client as bc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
# Path to IPC socket to use for web3 connections
|
||||
export BUGOUT_BROOD_URL="https://auth.bugout.dev"
|
||||
export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
|
||||
export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1"
|
||||
export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545"
|
||||
export MOONSTREAM_NODE_POLYGON_IPC_ADDR="127.0.0.1"
|
||||
|
@ -7,6 +8,8 @@ export MOONSTREAM_CRAWL_WORKERS=4
|
|||
export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
|
||||
export MOONSTREAM_ETHERSCAN_TOKEN="<Token for etherscan>"
|
||||
export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket for smart contracts>"
|
||||
export AWS_S3_SMARTCONTRACTS_ABI_BUCKET="<AWS S3 bucket to store smart contracts ABI>"
|
||||
export AWS_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx for AWS S3 bucket (v1/v2/dev/..)>"
|
||||
export MOONSTREAM_HUMBUG_TOKEN="<Token for crawlers store data via Humbug>"
|
||||
export COINMARKETCAP_API_KEY="<API key to parse conmarketcap>"
|
||||
export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"
|
||||
|
|
|
@ -36,11 +36,14 @@ setup(
|
|||
"boto3",
|
||||
"bugout",
|
||||
"chardet",
|
||||
"fastapi",
|
||||
"moonstreamdb",
|
||||
"humbug",
|
||||
"pydantic",
|
||||
"python-dateutil",
|
||||
"requests",
|
||||
"tqdm",
|
||||
"uvicorn",
|
||||
"web3",
|
||||
],
|
||||
extras_require={
|
||||
|
|
Ładowanie…
Reference in New Issue