Merge branch 'main' into status-page

backend/moonstream/api.py

@@ -12,6 +12,7 @@ from . import data
 from .middleware import MoonstreamHTTPException
 from .routes.address_info import app as addressinfo_api
 from .routes.nft import app as nft_api
+from .routes.whales import app as whales_api
 from .routes.subscriptions import app as subscriptions_api
 from .routes.streams import app as streams_api
 from .routes.txinfo import app as txinfo_api
@@ -75,3 +76,4 @@ app.mount("/streams", streams_api)
 app.mount("/txinfo", txinfo_api)
 app.mount("/address_info", addressinfo_api)
 app.mount("/nft", nft_api)
+app.mount("/whales", whales_api)

backend/moonstream/providers/bugout.py

@@ -327,7 +327,7 @@ class EthereumTXPoolProvider(BugoutEventProvider):
         return subscriptions_filters
 
 
-class NftProvider(BugoutEventProvider):
+class PublicDataProvider(BugoutEventProvider):
     def __init__(
         self,
         event_type: str,
@@ -364,7 +364,7 @@ Shows the top 10 addresses active on the Ethereum blockchain over the last hour
 4. Amount (in WEI) received
 
 To restrict your queries to this provider, add a filter of \"type:ethereum_whalewatch\" to your query (query parameter: \"q\") on the /streams endpoint."""
-whalewatch_provider = BugoutEventProvider(
+whalewatch_provider = PublicDataProvider(
     event_type="ethereum_whalewatch",
     description=whalewatch_description,
     default_time_interval_seconds=310,
@@ -394,7 +394,7 @@ Currently, it summarizes the activities on the following NFT markets:
 
 This provider is currently not accessible for subscription. The data from this provider is publicly
 available at the /nft endpoint."""
-nft_summary_provider = NftProvider(
+nft_summary_provider = PublicDataProvider(
     event_type="nft_summary",
     description=nft_summary_description,
     # 40 blocks per summary, 15 seconds per block + 2 seconds wiggle room.

backend/moonstream/routes/whales.py

@@ -0,0 +1,93 @@
+"""
+Moonstream's /whales endpoints.
+
+These endpoints provide public access to whale watch summaries. No authentication required.
+"""
+from datetime import datetime
+import logging
+from typing import Optional
+
+from bugout.data import BugoutResource
+
+from fastapi import Depends, FastAPI, Query
+from moonstreamdb import db
+from fastapi.middleware.cors import CORSMiddleware
+from sqlalchemy.orm import Session
+
+from .. import data
+from ..providers.bugout import whalewatch_provider
+from ..settings import (
+    bugout_client,
+    DOCS_TARGET_PATH,
+    MOONSTREAM_ADMIN_ACCESS_TOKEN,
+    MOONSTREAM_DATA_JOURNAL_ID,
+    ORIGINS,
+)
+from ..stream_queries import StreamQuery
+from ..version import MOONSTREAM_VERSION
+
+logger = logging.getLogger(__name__)
+
+tags_metadata = [
+    {"name": "whales", "description": "Whales summaries"},
+]
+
+app = FastAPI(
+    title=f"Moonstream /whales API",
+    description="User, token and password handlers.",
+    version=MOONSTREAM_VERSION,
+    openapi_tags=tags_metadata,
+    openapi_url="/openapi.json",
+    docs_url=None,
+    redoc_url=f"/{DOCS_TARGET_PATH}",
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=ORIGINS,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/", tags=["whales"], response_model=data.GetEventsResponse)
+async def stream_handler(
+    start_time: int = Query(0),
+    end_time: Optional[int] = Query(None),
+    include_start: bool = Query(False),
+    include_end: bool = Query(False),
+    db_session: Session = Depends(db.yield_db_session),
+) -> data.GetEventsResponse:
+    """
+    Retrieves the list of whales spotted over given stream boundary
+
+    - **start_time**: Timestamp. Must be provided otherwise this request will hang
+    - **end_time**: Timestamp. Optional.
+    - **include_start** (string): is start_time inclusive or not
+    - **include_end** (string): is end_time inclusive or not
+    """
+    stream_boundary = data.StreamBoundary(
+        start_time=start_time,
+        end_time=end_time,
+        include_start=include_start,
+        include_end=include_end,
+    )
+
+    result = whalewatch_provider.get_events(
+        db_session=db_session,
+        bugout_client=bugout_client,
+        data_journal_id=MOONSTREAM_DATA_JOURNAL_ID,
+        data_access_token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+        stream_boundary=stream_boundary,
+        user_subscriptions={whalewatch_provider.event_type: []},
+        query=StreamQuery(subscription_types=[whalewatch_provider.event_type]),
+    )
+
+    if result is None:
+        return data.GetEventsResponse(stream_boundary=stream_boundary, events=[])
+
+    provider_stream_boundary, events = result
+    return data.GetEventsResponse(
+        stream_boundary=provider_stream_boundary, events=events
+    )

crawlers/server/main.go

@@ -27,9 +27,17 @@ type PingResponse struct {
 	Status string `json:"status"`
 }
 
+// Extends handler with allowed CORS policies
 func setupCorsResponse(w *http.ResponseWriter, req *http.Request) {
-	(*w).Header().Set("Access-Control-Allow-Origin", MOONSTREAM_CORS_ALLOWED_ORIGINS)
-	(*w).Header().Set("Access-Control-Allow-Methods", "GET")
+	for _, allowedOrigin := range strings.Split(MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") {
+		for _, reqOrigin := range req.Header["Origin"] {
+			if reqOrigin == allowedOrigin {
+				(*w).Header().Set("Access-Control-Allow-Origin", allowedOrigin)
+			}
+		}
+
+	}
+	(*w).Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS")
 }
 
 func ping(w http.ResponseWriter, req *http.Request) {

db/server/main.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"strings"
 
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
@@ -22,9 +23,17 @@ type BlockResponse struct {
 	BlockNumber uint64 `json:"block_number"`
 }
 
+// Extends handler with allowed CORS policies
 func setupCorsResponse(w *http.ResponseWriter, req *http.Request) {
-	(*w).Header().Set("Access-Control-Allow-Origin", MOONSTREAM_CORS_ALLOWED_ORIGINS)
-	(*w).Header().Set("Access-Control-Allow-Methods", "GET")
+	for _, allowedOrigin := range strings.Split(MOONSTREAM_CORS_ALLOWED_ORIGINS, ",") {
+		for _, reqOrigin := range req.Header["Origin"] {
+			if reqOrigin == allowedOrigin {
+				(*w).Header().Set("Access-Control-Allow-Origin", allowedOrigin)
+			}
+		}
+
+	}
+	(*w).Header().Set("Access-Control-Allow-Methods", "GET,OPTIONS")
 }
 
 func ping(w http.ResponseWriter, req *http.Request) {