mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

CORS origin now is one resource 

This structure will help to track when origin were added.
pull/821/head
kompotkot 2023-06-22 13:16:48 +00:00
rodzic aca575052b
commit 5eca0fb57f
5 zmienionych plików z 183 dodań i 114 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

14
engineapi/engineapi/data.py
Wyświetl plik

@ -1,8 +1,9 @@
from datetime import datetime
from enum import Enum
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Set
from uuid import UUID
from bugout.data import BugoutResource
from pydantic import AnyHttpUrl, BaseModel, Field, root_validator, validator
from web3 import Web3
@ -23,8 +24,15 @@ class NowResponse(BaseModel):
epoch_time: float
class CORSResponse(BaseModel):
cors: List[AnyHttpUrl] = Field(default_factory=list)
class CORSOrigins(BaseModel):
origins_set: Set[str] = Field(default_factory=set)
resources: List[BugoutResource] = Field(default_factory=list)
class IsCORSResponse(BaseModel):
origin: Optional[str] = None
updated_at: Optional[datetime] = None
created_at: Optional[datetime] = None
class SignerListResponse(BaseModel):

152
engineapi/engineapi/middleware.py
Wyświetl plik

@ -1,9 +1,10 @@
import base64
import json
import logging
from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Set
from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Set, Tuple
from uuid import UUID
from bugout.data import BugoutResources, BugoutUser
from bugout.data import BugoutResource, BugoutResources, BugoutUser
from bugout.exceptions import BugoutResponseException
from fastapi import HTTPException, Request, Response
from pydantic import AnyHttpUrl, parse_obj_as
@ -13,6 +14,7 @@ from starlette.responses import Response
from starlette.types import ASGIApp
from web3 import Web3
from . import data
from .auth import (
MoonstreamAuthorizationExpired,
MoonstreamAuthorizationVerificationError,
@ -214,35 +216,73 @@ class ExtractBearerTokenMiddleware(BaseHTTPMiddleware):
return await call_next(request)
def parse_origins_from_resources(origins: List[str]) -> Set[str]:
def parse_origins_from_resources(
resources: List[BugoutResources],
) -> data.CORSOrigins:
"""
Parse list of CORS origins with HTTP validation and remove duplications.
"""
resource_origins_set = set()
for resource in origins:
origins = resource.resource_data.get("origins", [])
for o in origins:
try:
parse_obj_as(AnyHttpUrl, o)
resource_origins_set.add(o)
except Exception:
logger.info(f"Unable to parse origin: {o} as URL")
continue
cors_origins = data.CORSOrigins(origins_set=set())
for resource in resources:
origin = resource.resource_data.get("origin", "")
try:
parse_obj_as(AnyHttpUrl, origin)
cors_origins.origins_set.add(origin)
cors_origins.resources.append(resource)
except Exception:
logger.warning(
f"Unable to parse origin: {origin} as URL from resource {resource.id}"
)
continue
return resource_origins_set
return cors_origins
def check_default_origins(origins: Set[str]) -> Set[str]:
def check_default_origins(cors_origins: data.CORSOrigins) -> data.CORSOrigins:
"""
To prevent default origins loss.
"""
for o in ALLOW_ORIGINS:
if o not in origins:
origins.add(o)
return origins
if o not in cors_origins.origins_set:
cors_origins.origins_set.add(o)
return cors_origins
def fetch_application_settings_cors_origins(token: str) -> Set[str]:
def create_application_settings_cors_origin(
token: str, user_id: Tuple[str, UUID], username: str, origin: str
) -> Optional[BugoutResource]:
resource: Optional[BugoutResource] = None
try:
resource = bc.create_resource(
token=token,
application_id=MOONSTREAM_APPLICATION_ID,
resource_data={
"type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG,
"setting": "cors",
"user_id": str(user_id),
"username": username,
"origin": origin,
},
)
if token != MOONSTREAM_ADMIN_ACCESS_TOKEN:
bc.add_resource_holder_permissions(
token=token,
resource_id=resource.id,
holder_permissions={
"holder_id": str(MOONSTREAM_ADMIN_USER.id),
"holder_type": "user",
"permissions": ["admin", "create", "read", "update", "delete"],
},
)
except Exception as err:
logger.error(
f"Unable to write default CORS origin {origin} to Brood resource: {str(err)}"
)
return resource
def fetch_application_settings_cors_origins(token: str) -> data.CORSOrigins:
"""
Fetch application config resources with CORS origins setting.
If there are no such resources create new one with default origins from environment variable.
@ -265,54 +305,58 @@ def fetch_application_settings_cors_origins(token: str) -> Set[str]:
except Exception as err:
logger.error(f"Error fetching bugout resources with CORS origins: {str(err)}")
return ALLOW_ORIGINS
return data.CORSOrigins(origins_set=ALLOW_ORIGINS)
# If there are no resources with CORS origins configuration, create new one
# with default list of origins from environment variable
# If there are no resources with CORS origins configuration, create resources
# for each default origin from environment variable
if len(resources.resources) == 0:
try:
resource = bc.create_resource(
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
application_id=MOONSTREAM_APPLICATION_ID,
resource_data={
"type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG,
"setting": "cors",
"user_id": str(MOONSTREAM_ADMIN_USER.id),
"origins": ALLOW_ORIGINS,
},
)
resources.resources.append(resource)
logger.info(
"Created resource with default CORS origins setting by moonstream admin user"
)
except Exception as err:
logger.error(
f"Unable to write default CORS origins to resources: {str(err)}"
)
return ALLOW_ORIGINS
default_origins_cnt = 0
for o in ALLOW_ORIGINS:
# Try to add new origins to Bugout resources application config,
# use 3 retries to assure origin added and not passed because of some network error.
retry_cnt = 0
while retry_cnt < 3:
resource = create_application_settings_cors_origin(
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
user_id=str(MOONSTREAM_ADMIN_USER.id),
username=MOONSTREAM_ADMIN_USER.username,
origin=o,
)
if resource is not None:
resources.resources.append(resource)
default_origins_cnt += 1
break
retry_cnt += 1
resource_origins_set: Set[str] = parse_origins_from_resources(resources.resources)
resource_origins_set = check_default_origins(resource_origins_set)
if default_origins_cnt != len(ALLOW_ORIGINS):
return data.CORSOrigins(origins_set=ALLOW_ORIGINS)
return list(resource_origins_set)
logger.info(
f"Created resources with default {default_origins_cnt} CORS origins setting by moonstream admin user"
)
cors_origins: data.CORSOrigins = parse_origins_from_resources(resources.resources)
cors_origins = check_default_origins(cors_origins)
return cors_origins
def set_cors_origins_cache(allow_origins: Set[str]) -> None:
def set_cors_origins_cache(origins_set: Set[str]) -> None:
try:
rc_client.sadd(REDIS_CONFIG_CORS_KEY, *allow_origins)
rc_client.sadd(REDIS_CONFIG_CORS_KEY, *origins_set)
except Exception:
logger.warning("Unable to set CORS origins at Redis cache")
finally:
rc_client.close()
def fetch_and_set_cors_origins_cache():
allow_origins = fetch_application_settings_cors_origins(
def fetch_and_set_cors_origins_cache() -> data.CORSOrigins:
cors_origins = fetch_application_settings_cors_origins(
token=MOONSTREAM_ADMIN_ACCESS_TOKEN
)
set_cors_origins_cache(allow_origins)
set_cors_origins_cache(cors_origins.origins_set)
return list(allow_origins)
return cors_origins
class BugoutCORSMiddleware(CORSMiddleware):
@ -330,11 +374,13 @@ class BugoutCORSMiddleware(CORSMiddleware):
expose_headers: Sequence[str] = (),
max_age: int = 600,
):
application_configs_allowed_origins = fetch_and_set_cors_origins_cache()
application_configs_allowed_origins: data.CORSOrigins = (
fetch_and_set_cors_origins_cache()
)
super().__init__(
app=app,
allow_origins=application_configs_allowed_origins,
allow_origins=list(application_configs_allowed_origins.origins_set),
allow_methods=allow_methods,
allow_headers=allow_headers,
allow_credentials=allow_credentials,

122
engineapi/engineapi/routes/configs.py
Wyświetl plik

@ -1,24 +1,25 @@
import logging
from typing import Any, Dict, List, Set
from bugout.data import BugoutResource
from bugout.data import BugoutResource, BugoutResources
from fastapi import (
BackgroundTasks,
Body,
Depends,
FastAPI,
Form,
HTTPException,
Query,
Request,
)
from pydantic import AnyHttpUrl
from .. import actions, data
from .. import data
from ..middleware import (
BroodAuthMiddleware,
BugoutCORSMiddleware,
EngineHTTPException,
check_default_origins,
create_application_settings_cors_origin,
fetch_and_set_cors_origins_cache,
parse_origins_from_resources,
)
@ -26,6 +27,7 @@ from ..settings import (
BUGOUT_REQUEST_TIMEOUT_SECONDS,
BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG,
DOCS_TARGET_PATH,
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_ADMIN_USER,
MOONSTREAM_APPLICATION_ID,
)
@ -44,6 +46,7 @@ whitelist_paths.update(
{
"/configs/docs": "GET",
"/configs/openapi.json": "GET",
"/configs/is_origin": "GET",
}
)
@ -68,10 +71,42 @@ app.add_middleware(
)
@app.get("/cors", response_model=data.CORSResponse)
async def get_cors(
@app.get("/is_origin", response_model=data.IsCORSResponse)
async def is_cors_origin(origin: str = Query(...)) -> data.IsCORSResponse:
is_cors_origin = data.IsCORSResponse()
try:
resources = bc.list_resources(
token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
params={
"application_id": MOONSTREAM_APPLICATION_ID,
"type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG,
"setting": "cors",
},
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
cors_origins: data.CORSOrigins = parse_origins_from_resources(
resources.resources
)
if origin in cors_origins.origins_set:
for resource in cors_origins.resources:
resource_origin = resource.resource_data.get("origin", "")
# TODO(kompotkot): There are could be multiple creations by different users.
# Add logic to show most recent updated_at and oldest created_at.
if resource_origin == origin:
is_cors_origin.origin = resource_origin
is_cors_origin.created_at = resource.created_at
is_cors_origin.updated_at = resource.updated_at
except Exception as err:
logger.error(repr(err))
raise EngineHTTPException(status_code=500)
return is_cors_origin
@app.get("/origins", response_model=data.CORSOrigins)
async def get_cors_origins(
request: Request,
) -> data.CORSResponse:
) -> data.CORSOrigins:
try:
resources = bc.list_resources(
token=request.state.token,
@ -82,27 +117,23 @@ async def get_cors(
},
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
resource_origins_set: Set[str] = parse_origins_from_resources(
cors_origins: data.CORSOrigins = parse_origins_from_resources(
resources.resources
)
except Exception as err:
logger.error(repr(err))
raise EngineHTTPException(status_code=500)
return data.CORSResponse(cors=list(resource_origins_set))
return cors_origins
@app.put("/cors", response_model=data.CORSResponse)
async def update_cors(
@app.post("/origin", response_model=data.CORSOrigins)
async def add_cors_origin(
request: Request,
background_tasks: BackgroundTasks,
new_origins: List[AnyHttpUrl] = Body(...),
) -> data.CORSResponse:
new_origins = set(new_origins)
new_origin: AnyHttpUrl = Form(...),
) -> data.CORSOrigins:
try:
target_resource: BugoutResource
resources = bc.list_resources(
token=request.state.token,
params={
@ -112,50 +143,29 @@ async def update_cors(
},
timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
)
if len(resources.resources) == 0:
target_resource = bc.create_resource(
token=request.state.token,
application_id=MOONSTREAM_APPLICATION_ID,
resource_data={
"type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG,
"setting": "cors",
"user_id": str(request.state.user.id),
"origins": list(new_origins),
},
)
bc.add_resource_holder_permissions(
token=request.state.token,
resource_id=target_resource.id,
holder_permissions={
"holder_id": str(MOONSTREAM_ADMIN_USER.id),
"holder_type": "user",
"permissions": ["admin", "create", "read", "update", "delete"],
},
)
elif len(resources.resources) == 1:
target_resource = resources.resources[0]
resource_origins_set: Set[str] = parse_origins_from_resources(
[target_resource]
)
resource_origins_set.update(new_origins)
target_resource = bc.update_resource(
token=request.state.token,
resource_id=target_resource.id,
resource_data={
"update": {"origins": list(resource_origins_set)},
"drop_keys": [],
},
)
elif len(resources.resources) > 1:
# TODO(kompotkot): Remove all resource and save only one
raise EngineHTTPException(status_code=500)
except Exception as err:
logger.error(repr(err))
logger.error(f"Unable to fetch resource from Brood, err: {repr(err)}")
raise EngineHTTPException(status_code=500)
cors_origins: data.CORSOrigins = parse_origins_from_resources(resources.resources)
if new_origin in cors_origins.origins_set:
raise EngineHTTPException(
status_code=409,
detail=f"Provided origin {new_origin} already set by user",
)
resource = create_application_settings_cors_origin(
token=request.state.token,
user_id=request.state.user.id,
username=request.state.user.username,
origin=new_origin,
)
cors_origins.origins_set.add(new_origin)
cors_origins.resources.append(resource)
background_tasks.add_task(
fetch_and_set_cors_origins_cache,
)
return data.CORSResponse(cors=target_resource.resource_data["origins"])
return cors_origins

8
engineapi/engineapi/settings.py
Wyświetl plik

@ -1,7 +1,7 @@
import logging
import os
import warnings
from typing import List
from typing import Set
from bugout.app import Bugout
from bugout.data import BugoutUser
@ -26,7 +26,11 @@ if RAW_ORIGINS is None:
raise ValueError(
"ENGINE_CORS_ALLOWED_ORIGINS environment variable must be set (comma-separated list of CORS allowed origins)"
)
ALLOW_ORIGINS: List[str] = RAW_ORIGINS.split(",")
RAW_ORIGINS_LST = RAW_ORIGINS.split(",")
ALLOW_ORIGINS: Set[str] = set()
for o_raw in RAW_ORIGINS_LST:
ALLOW_ORIGINS.add(o_raw.strip())
BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG = "application-config"
BUGOUT_REQUEST_TIMEOUT_SECONDS = 5

1
engineapi/setup.py
Wyświetl plik

@ -20,6 +20,7 @@ setup(
"redis",
"psycopg2-binary",
"pydantic",
"python-multipart",
"sqlalchemy",
"tqdm",
"uvicorn",