API endpoint for crawlers

crawlers/mooncrawl/mooncrawl/api.py

@@ -0,0 +1,81 @@
+"""
+The Mooncrawl HTTP API
+"""
+import logging
+import time
+from typing import Dict
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from . import data
+from .middleware import MoonstreamHTTPException
+from .settings import DOCS_TARGET_PATH, ORIGINS
+from .version import MOONCRAWL_VERSION
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+tags_metadata = [
+    {"name": "jobs", "description": "Trigger crawler jobs."},
+    {"name": "time", "description": "Server timestamp endpoints."},
+]
+
+app = FastAPI(
+    title=f"Mooncrawl HTTP API",
+    description="Mooncrawl API endpoints.",
+    version=MOONCRAWL_VERSION,
+    openapi_tags=tags_metadata,
+    openapi_url="/openapi.json",
+    docs_url=None,
+    redoc_url=f"/{DOCS_TARGET_PATH}",
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=ORIGINS,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/ping", response_model=data.PingResponse)
+async def ping_handler() -> data.PingResponse:
+    """
+    Check server status.
+    """
+    return data.PingResponse(status="ok")
+
+
+@app.get("/version", response_model=data.VersionResponse)
+async def version_handler() -> data.VersionResponse:
+    """
+    Get server version.
+    """
+    return data.VersionResponse(version=MOONCRAWL_VERSION)
+
+
+@app.get("/now", tags=["time"])
+async def now_handler() -> data.NowResponse:
+    """
+    Get server current time.
+    """
+    return data.NowResponse(epoch_time=time.time())
+
+
+@app.get("/jobs/stats_update", tags=["jobs"])
+async def status_handler():
+    """
+    Find latest crawlers records with creation timestamp:
+    - ethereum_txpool
+    - ethereum_trending
+    """
+    try:
+        pass
+    except Exception as e:
+        logger.error(f"Unhandled status exception, error: {e}")
+        raise MoonstreamHTTPException(status_code=500)
+
+    return

crawlers/mooncrawl/mooncrawl/data.py

@@ -2,6 +2,8 @@ from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
 
+from pydantic import BaseModel
+
 
 class AvailableBlockchainType(Enum):
     ETHEREUM = "ethereum"
@@ -14,3 +16,27 @@ class DateRange:
     end_time: datetime
     include_start: bool
     include_end: bool
+
+
+class PingResponse(BaseModel):
+    """
+    Schema for ping response
+    """
+
+    status: str
+
+
+class VersionResponse(BaseModel):
+    """
+    Schema for responses on /version endpoint
+    """
+
+    version: str
+
+
+class NowResponse(BaseModel):
+    """
+    Schema for responses on /now endpoint
+    """
+
+    epoch_time: float

crawlers/mooncrawl/mooncrawl/middleware.py

@@ -0,0 +1,26 @@
+import logging
+from typing import Any, Dict, Optional
+
+from fastapi import HTTPException
+
+from .reporter import reporter
+
+logger = logging.getLogger(__name__)
+
+
+class MoonstreamHTTPException(HTTPException):
+    """
+    Extended HTTPException to handle 500 Internal server errors
+    and send crash reports.
+    """
+
+    def __init__(
+        self,
+        status_code: int,
+        detail: Any = None,
+        headers: Optional[Dict[str, Any]] = None,
+        internal_error: Exception = None,
+    ):
+        super().__init__(status_code, detail, headers)
+        if internal_error is not None:
+            reporter.error_report(internal_error)

crawlers/mooncrawl/mooncrawl/settings.py

@@ -6,10 +6,21 @@ from bugout.app import Bugout
 # Bugout
 BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev")
 BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev")
-bc = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL)
+bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL)
 
 HUMBUG_REPORTER_CRAWLERS_TOKEN = os.environ.get("HUMBUG_REPORTER_CRAWLERS_TOKEN")
 
+# Origin
+RAW_ORIGINS = os.environ.get("MOONSTREAM_CORS_ALLOWED_ORIGINS")
+if RAW_ORIGINS is None:
+    raise ValueError(
+        "MOONSTREAM_CORS_ALLOWED_ORIGINS environment variable must be set (comma-separated list of CORS allowed origins)"
+    )
+ORIGINS = RAW_ORIGINS.split(",")
+
+# OpenAPI
+DOCS_TARGET_PATH = "docs"
+
 # Geth connection address
 MOONSTREAM_NODE_ETHEREUM_IPC_ADDR = os.environ.get(
     "MOONSTREAM_NODE_ETHEREUM_IPC_ADDR", "127.0.0.1"
@@ -56,4 +67,15 @@ MOONSTREAM_DATA_JOURNAL_ID = os.environ.get("MOONSTREAM_DATA_JOURNAL_ID", "")
 if MOONSTREAM_DATA_JOURNAL_ID == "":
     raise ValueError("MOONSTREAM_DATA_JOURNAL_ID env variable is not set")
 
-AWS_S3_SMARTCONTRACTS_ABI_PREFIX = os.getenv("AWS_S3_SMARTCONTRACTS_ABI_PREFIX")
+# AWS S3 bucket for ABI smartcontract settings
+AWS_S3_SMARTCONTRACTS_ABI_BUCKET = os.environ.get("AWS_S3_SMARTCONTRACTS_ABI_BUCKET")
+if AWS_S3_SMARTCONTRACTS_ABI_BUCKET is None:
+    raise ValueError(
+        "AWS_S3_SMARTCONTRACTS_ABI_BUCKET environment variable must be set"
+    )
+AWS_S3_SMARTCONTRACTS_ABI_PREFIX = os.environ.get("AWS_S3_SMARTCONTRACTS_ABI_PREFIX")
+if AWS_S3_SMARTCONTRACTS_ABI_PREFIX is None:
+    raise ValueError(
+        "AWS_S3_SMARTCONTRACTS_ABI_PREFIX environment variable must be set"
+    )
+AWS_S3_SMARTCONTRACTS_ABI_PREFIX = AWS_S3_SMARTCONTRACTS_ABI_PREFIX.rstrip("/")

crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py

@@ -19,10 +19,11 @@ from sqlalchemy.orm import Query, Session
 from ..blockchain import get_block_model, get_label_model, get_transaction_model
 from ..data import AvailableBlockchainType
 from ..settings import (
+    AWS_S3_SMARTCONTRACTS_ABI_BUCKET,
     AWS_S3_SMARTCONTRACTS_ABI_PREFIX,
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
-    bc,
 )
+from ..settings import bugout_client as bc
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)

crawlers/mooncrawl/sample.env

@@ -1,4 +1,5 @@
-# Path to IPC socket to use for web3 connections
+export BUGOUT_BROOD_URL="https://auth.bugout.dev"
+export BUGOUT_SPIRE_URL="https://spire.bugout.dev"
 export MOONSTREAM_NODE_ETHEREUM_IPC_ADDR="127.0.0.1"
 export MOONSTREAM_NODE_ETHEREUM_IPC_PORT="8545"
 export MOONSTREAM_NODE_POLYGON_IPC_ADDR="127.0.0.1"
@@ -7,6 +8,8 @@ export MOONSTREAM_CRAWL_WORKERS=4
 export MOONSTREAM_DB_URI="postgresql://<username>:<password>@<db_host>:<db_port>/<db_name>"
 export MOONSTREAM_ETHERSCAN_TOKEN="<Token for etherscan>"
 export AWS_S3_SMARTCONTRACT_BUCKET="<AWS S3 bucket for smart contracts>"
+export AWS_S3_SMARTCONTRACTS_ABI_BUCKET="<AWS S3 bucket to store smart contracts ABI>"
+export AWS_S3_SMARTCONTRACTS_ABI_PREFIX="<Previx for AWS S3 bucket (v1/v2/dev/..)>"
 export MOONSTREAM_HUMBUG_TOKEN="<Token for crawlers store data via Humbug>"
 export COINMARKETCAP_API_KEY="<API key to parse conmarketcap>"
 export HUMBUG_REPORTER_CRAWLERS_TOKEN="<Bugout Humbug token for crash reports>"

crawlers/mooncrawl/setup.py

@@ -36,11 +36,14 @@ setup(
         "boto3",
         "bugout",
         "chardet",
+        "fastapi",
         "moonstreamdb",
         "humbug",
+        "pydantic",
         "python-dateutil",
         "requests",
         "tqdm",
+        "uvicorn",
         "web3",
     ],
     extras_require={