diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 1a40b6c6..09064b93 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -1,8 +1,9 @@ from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set from uuid import UUID +from bugout.data import BugoutResource from pydantic import AnyHttpUrl, BaseModel, Field, root_validator, validator from web3 import Web3 @@ -23,8 +24,15 @@ class NowResponse(BaseModel): epoch_time: float -class CORSResponse(BaseModel): - cors: List[AnyHttpUrl] = Field(default_factory=list) +class CORSOrigins(BaseModel): + origins_set: Set[str] = Field(default_factory=set) + resources: List[BugoutResource] = Field(default_factory=list) + + +class IsCORSResponse(BaseModel): + origin: Optional[str] = None + updated_at: Optional[datetime] = None + created_at: Optional[datetime] = None class SignerListResponse(BaseModel): diff --git a/engineapi/engineapi/middleware.py b/engineapi/engineapi/middleware.py index ccbd822f..d43fcf34 100644 --- a/engineapi/engineapi/middleware.py +++ b/engineapi/engineapi/middleware.py @@ -1,9 +1,10 @@ import base64 import json import logging -from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Set +from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Set, Tuple +from uuid import UUID -from bugout.data import BugoutResources, BugoutUser +from bugout.data import BugoutResource, BugoutResources, BugoutUser from bugout.exceptions import BugoutResponseException from fastapi import HTTPException, Request, Response from pydantic import AnyHttpUrl, parse_obj_as @@ -13,6 +14,7 @@ from starlette.responses import Response from starlette.types import ASGIApp from web3 import Web3 +from . import data from .auth import ( MoonstreamAuthorizationExpired, MoonstreamAuthorizationVerificationError, @@ -214,35 +216,73 @@ class ExtractBearerTokenMiddleware(BaseHTTPMiddleware): return await call_next(request) -def parse_origins_from_resources(origins: List[str]) -> Set[str]: +def parse_origins_from_resources( + resources: List[BugoutResources], +) -> data.CORSOrigins: """ Parse list of CORS origins with HTTP validation and remove duplications. """ - resource_origins_set = set() - for resource in origins: - origins = resource.resource_data.get("origins", []) - for o in origins: - try: - parse_obj_as(AnyHttpUrl, o) - resource_origins_set.add(o) - except Exception: - logger.info(f"Unable to parse origin: {o} as URL") - continue + cors_origins = data.CORSOrigins(origins_set=set()) + for resource in resources: + origin = resource.resource_data.get("origin", "") + try: + parse_obj_as(AnyHttpUrl, origin) + cors_origins.origins_set.add(origin) + cors_origins.resources.append(resource) + except Exception: + logger.warning( + f"Unable to parse origin: {origin} as URL from resource {resource.id}" + ) + continue - return resource_origins_set + return cors_origins -def check_default_origins(origins: Set[str]) -> Set[str]: +def check_default_origins(cors_origins: data.CORSOrigins) -> data.CORSOrigins: """ To prevent default origins loss. """ for o in ALLOW_ORIGINS: - if o not in origins: - origins.add(o) - return origins + if o not in cors_origins.origins_set: + cors_origins.origins_set.add(o) + return cors_origins -def fetch_application_settings_cors_origins(token: str) -> Set[str]: +def create_application_settings_cors_origin( + token: str, user_id: Tuple[str, UUID], username: str, origin: str +) -> Optional[BugoutResource]: + resource: Optional[BugoutResource] = None + try: + resource = bc.create_resource( + token=token, + application_id=MOONSTREAM_APPLICATION_ID, + resource_data={ + "type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, + "setting": "cors", + "user_id": str(user_id), + "username": username, + "origin": origin, + }, + ) + if token != MOONSTREAM_ADMIN_ACCESS_TOKEN: + bc.add_resource_holder_permissions( + token=token, + resource_id=resource.id, + holder_permissions={ + "holder_id": str(MOONSTREAM_ADMIN_USER.id), + "holder_type": "user", + "permissions": ["admin", "create", "read", "update", "delete"], + }, + ) + except Exception as err: + logger.error( + f"Unable to write default CORS origin {origin} to Brood resource: {str(err)}" + ) + + return resource + + +def fetch_application_settings_cors_origins(token: str) -> data.CORSOrigins: """ Fetch application config resources with CORS origins setting. If there are no such resources create new one with default origins from environment variable. @@ -265,54 +305,58 @@ def fetch_application_settings_cors_origins(token: str) -> Set[str]: except Exception as err: logger.error(f"Error fetching bugout resources with CORS origins: {str(err)}") - return ALLOW_ORIGINS + return data.CORSOrigins(origins_set=ALLOW_ORIGINS) - # If there are no resources with CORS origins configuration, create new one - # with default list of origins from environment variable + # If there are no resources with CORS origins configuration, create resources + # for each default origin from environment variable if len(resources.resources) == 0: - try: - resource = bc.create_resource( - token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - application_id=MOONSTREAM_APPLICATION_ID, - resource_data={ - "type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, - "setting": "cors", - "user_id": str(MOONSTREAM_ADMIN_USER.id), - "origins": ALLOW_ORIGINS, - }, - ) - resources.resources.append(resource) - logger.info( - "Created resource with default CORS origins setting by moonstream admin user" - ) - except Exception as err: - logger.error( - f"Unable to write default CORS origins to resources: {str(err)}" - ) - return ALLOW_ORIGINS + default_origins_cnt = 0 + for o in ALLOW_ORIGINS: + # Try to add new origins to Bugout resources application config, + # use 3 retries to assure origin added and not passed because of some network error. + retry_cnt = 0 + while retry_cnt < 3: + resource = create_application_settings_cors_origin( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + user_id=str(MOONSTREAM_ADMIN_USER.id), + username=MOONSTREAM_ADMIN_USER.username, + origin=o, + ) + if resource is not None: + resources.resources.append(resource) + default_origins_cnt += 1 + break + retry_cnt += 1 - resource_origins_set: Set[str] = parse_origins_from_resources(resources.resources) - resource_origins_set = check_default_origins(resource_origins_set) + if default_origins_cnt != len(ALLOW_ORIGINS): + return data.CORSOrigins(origins_set=ALLOW_ORIGINS) - return list(resource_origins_set) + logger.info( + f"Created resources with default {default_origins_cnt} CORS origins setting by moonstream admin user" + ) + + cors_origins: data.CORSOrigins = parse_origins_from_resources(resources.resources) + cors_origins = check_default_origins(cors_origins) + + return cors_origins -def set_cors_origins_cache(allow_origins: Set[str]) -> None: +def set_cors_origins_cache(origins_set: Set[str]) -> None: try: - rc_client.sadd(REDIS_CONFIG_CORS_KEY, *allow_origins) + rc_client.sadd(REDIS_CONFIG_CORS_KEY, *origins_set) except Exception: logger.warning("Unable to set CORS origins at Redis cache") finally: rc_client.close() -def fetch_and_set_cors_origins_cache(): - allow_origins = fetch_application_settings_cors_origins( +def fetch_and_set_cors_origins_cache() -> data.CORSOrigins: + cors_origins = fetch_application_settings_cors_origins( token=MOONSTREAM_ADMIN_ACCESS_TOKEN ) - set_cors_origins_cache(allow_origins) + set_cors_origins_cache(cors_origins.origins_set) - return list(allow_origins) + return cors_origins class BugoutCORSMiddleware(CORSMiddleware): @@ -330,11 +374,13 @@ class BugoutCORSMiddleware(CORSMiddleware): expose_headers: Sequence[str] = (), max_age: int = 600, ): - application_configs_allowed_origins = fetch_and_set_cors_origins_cache() + application_configs_allowed_origins: data.CORSOrigins = ( + fetch_and_set_cors_origins_cache() + ) super().__init__( app=app, - allow_origins=application_configs_allowed_origins, + allow_origins=list(application_configs_allowed_origins.origins_set), allow_methods=allow_methods, allow_headers=allow_headers, allow_credentials=allow_credentials, diff --git a/engineapi/engineapi/routes/configs.py b/engineapi/engineapi/routes/configs.py index 12c0e460..351b4016 100644 --- a/engineapi/engineapi/routes/configs.py +++ b/engineapi/engineapi/routes/configs.py @@ -1,24 +1,25 @@ import logging from typing import Any, Dict, List, Set -from bugout.data import BugoutResource +from bugout.data import BugoutResource, BugoutResources from fastapi import ( BackgroundTasks, Body, Depends, FastAPI, + Form, HTTPException, Query, Request, ) from pydantic import AnyHttpUrl -from .. import actions, data +from .. import data from ..middleware import ( BroodAuthMiddleware, BugoutCORSMiddleware, EngineHTTPException, - check_default_origins, + create_application_settings_cors_origin, fetch_and_set_cors_origins_cache, parse_origins_from_resources, ) @@ -26,6 +27,7 @@ from ..settings import ( BUGOUT_REQUEST_TIMEOUT_SECONDS, BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, DOCS_TARGET_PATH, + MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_ADMIN_USER, MOONSTREAM_APPLICATION_ID, ) @@ -44,6 +46,7 @@ whitelist_paths.update( { "/configs/docs": "GET", "/configs/openapi.json": "GET", + "/configs/is_origin": "GET", } ) @@ -68,10 +71,42 @@ app.add_middleware( ) -@app.get("/cors", response_model=data.CORSResponse) -async def get_cors( +@app.get("/is_origin", response_model=data.IsCORSResponse) +async def is_cors_origin(origin: str = Query(...)) -> data.IsCORSResponse: + is_cors_origin = data.IsCORSResponse() + try: + resources = bc.list_resources( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + params={ + "application_id": MOONSTREAM_APPLICATION_ID, + "type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, + "setting": "cors", + }, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + cors_origins: data.CORSOrigins = parse_origins_from_resources( + resources.resources + ) + if origin in cors_origins.origins_set: + for resource in cors_origins.resources: + resource_origin = resource.resource_data.get("origin", "") + # TODO(kompotkot): There are could be multiple creations by different users. + # Add logic to show most recent updated_at and oldest created_at. + if resource_origin == origin: + is_cors_origin.origin = resource_origin + is_cors_origin.created_at = resource.created_at + is_cors_origin.updated_at = resource.updated_at + except Exception as err: + logger.error(repr(err)) + raise EngineHTTPException(status_code=500) + + return is_cors_origin + + +@app.get("/origins", response_model=data.CORSOrigins) +async def get_cors_origins( request: Request, -) -> data.CORSResponse: +) -> data.CORSOrigins: try: resources = bc.list_resources( token=request.state.token, @@ -82,27 +117,23 @@ async def get_cors( }, timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) - resource_origins_set: Set[str] = parse_origins_from_resources( + cors_origins: data.CORSOrigins = parse_origins_from_resources( resources.resources ) except Exception as err: logger.error(repr(err)) raise EngineHTTPException(status_code=500) - return data.CORSResponse(cors=list(resource_origins_set)) + return cors_origins -@app.put("/cors", response_model=data.CORSResponse) -async def update_cors( +@app.post("/origin", response_model=data.CORSOrigins) +async def add_cors_origin( request: Request, background_tasks: BackgroundTasks, - new_origins: List[AnyHttpUrl] = Body(...), -) -> data.CORSResponse: - new_origins = set(new_origins) - + new_origin: AnyHttpUrl = Form(...), +) -> data.CORSOrigins: try: - target_resource: BugoutResource - resources = bc.list_resources( token=request.state.token, params={ @@ -112,50 +143,29 @@ async def update_cors( }, timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) - if len(resources.resources) == 0: - target_resource = bc.create_resource( - token=request.state.token, - application_id=MOONSTREAM_APPLICATION_ID, - resource_data={ - "type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, - "setting": "cors", - "user_id": str(request.state.user.id), - "origins": list(new_origins), - }, - ) - bc.add_resource_holder_permissions( - token=request.state.token, - resource_id=target_resource.id, - holder_permissions={ - "holder_id": str(MOONSTREAM_ADMIN_USER.id), - "holder_type": "user", - "permissions": ["admin", "create", "read", "update", "delete"], - }, - ) - elif len(resources.resources) == 1: - target_resource = resources.resources[0] - resource_origins_set: Set[str] = parse_origins_from_resources( - [target_resource] - ) - resource_origins_set.update(new_origins) - - target_resource = bc.update_resource( - token=request.state.token, - resource_id=target_resource.id, - resource_data={ - "update": {"origins": list(resource_origins_set)}, - "drop_keys": [], - }, - ) - elif len(resources.resources) > 1: - # TODO(kompotkot): Remove all resource and save only one - raise EngineHTTPException(status_code=500) except Exception as err: - logger.error(repr(err)) + logger.error(f"Unable to fetch resource from Brood, err: {repr(err)}") raise EngineHTTPException(status_code=500) + cors_origins: data.CORSOrigins = parse_origins_from_resources(resources.resources) + + if new_origin in cors_origins.origins_set: + raise EngineHTTPException( + status_code=409, + detail=f"Provided origin {new_origin} already set by user", + ) + + resource = create_application_settings_cors_origin( + token=request.state.token, + user_id=request.state.user.id, + username=request.state.user.username, + origin=new_origin, + ) + cors_origins.origins_set.add(new_origin) + cors_origins.resources.append(resource) + background_tasks.add_task( fetch_and_set_cors_origins_cache, ) - return data.CORSResponse(cors=target_resource.resource_data["origins"]) + return cors_origins diff --git a/engineapi/engineapi/settings.py b/engineapi/engineapi/settings.py index f8be116f..bffad212 100644 --- a/engineapi/engineapi/settings.py +++ b/engineapi/engineapi/settings.py @@ -1,7 +1,7 @@ import logging import os import warnings -from typing import List +from typing import Set from bugout.app import Bugout from bugout.data import BugoutUser @@ -26,7 +26,11 @@ if RAW_ORIGINS is None: raise ValueError( "ENGINE_CORS_ALLOWED_ORIGINS environment variable must be set (comma-separated list of CORS allowed origins)" ) -ALLOW_ORIGINS: List[str] = RAW_ORIGINS.split(",") +RAW_ORIGINS_LST = RAW_ORIGINS.split(",") +ALLOW_ORIGINS: Set[str] = set() +for o_raw in RAW_ORIGINS_LST: + ALLOW_ORIGINS.add(o_raw.strip()) + BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG = "application-config" BUGOUT_REQUEST_TIMEOUT_SECONDS = 5 diff --git a/engineapi/setup.py b/engineapi/setup.py index 9fa55d56..e8db532a 100644 --- a/engineapi/setup.py +++ b/engineapi/setup.py @@ -20,6 +20,7 @@ setup( "redis", "psycopg2-binary", "pydantic", + "python-multipart", "sqlalchemy", "tqdm", "uvicorn",