diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 079148ba..974a3ecb 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -92,7 +92,6 @@ def create_dropper_contract( def delete_dropper_contract( db_session: Session, blockchain: Optional[str], dropper_contract_address ): - dropper_contract = ( db_session.query(DropperContract) .filter( @@ -877,7 +876,6 @@ def refetch_drop_signatures( for outdated_signature, transformed_claim_amount in zip( page, transformed_claim_amounts ): - message_hash_raw = dropper_contract.claimMessageHash( claim.claim_id, outdated_signature.address, @@ -1174,7 +1172,6 @@ def add_scores( addresses = [score.address for score in scores] if len(addresses) != len(set(addresses)): - duplicates = [key for key, value in Counter(addresses).items() if value > 1] raise DuplicateLeaderboardAddressError("Dublicated addresses", duplicates) @@ -1225,7 +1222,6 @@ def create_leaderboard_resource( leaderboard_id: uuid.UUID, token: Optional[uuid.UUID] = None, ) -> BugoutResource: - resource_data: Dict[str, Any] = { "type": LEADERBOARD_RESOURCE_TYPE, "leaderboard_id": leaderboard_id, @@ -1248,7 +1244,6 @@ def assign_resource( leaderboard_id: uuid.UUID, resource_id: Optional[uuid.UUID] = None, ): - """ Assign a resource handler to a leaderboard """ @@ -1258,7 +1253,6 @@ def assign_resource( ) if leaderboard.resource_id is not None: - raise Exception("Leaderboard already has a resource") if resource_id is not None: @@ -1281,7 +1275,6 @@ def assign_resource( def list_leaderboards_resources( db_session: Session, ): - """ List all leaderboards resources """ @@ -1292,7 +1285,6 @@ def list_leaderboards_resources( def revoke_resource(db_session: Session, leaderboard_id: uuid.UUID): - """ Revoke a resource handler to a leaderboard """ @@ -1304,7 +1296,6 @@ def revoke_resource(db_session: Session, leaderboard_id: uuid.UUID): ) if leaderboard.resource_id is None: - raise Exception("Leaderboard does not have a resource") leaderboard.resource_id = None diff --git a/engineapi/engineapi/api.py b/engineapi/engineapi/api.py index 39aba830..39876919 100644 --- a/engineapi/engineapi/api.py +++ b/engineapi/engineapi/api.py @@ -5,17 +5,15 @@ import logging import time from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware from . import data -from .settings import ( - ORIGINS, -) +from .middleware import BugoutCORSMiddleware +from .routes.admin import app as admin_app +from .routes.configs import app as configs_app from .routes.dropper import app as dropper_app from .routes.leaderboard import app as leaderboard_app -from .routes.admin import app as admin_app -from .routes.play import app as play_app from .routes.metatx import app as metatx_app +from .routes.play import app as play_app from .version import VERSION logging.basicConfig(level=logging.INFO) @@ -34,8 +32,7 @@ app = FastAPI( ) app.add_middleware( - CORSMiddleware, - allow_origins=ORIGINS, + BugoutCORSMiddleware, allow_credentials=True, allow_methods=["*"], allow_headers=["*"], @@ -58,8 +55,9 @@ async def now_handler() -> data.NowResponse: return data.NowResponse(epoch_time=time.time()) +app.mount("/admin", admin_app) +app.mount("/configs", configs_app) app.mount("/leaderboard", leaderboard_app) app.mount("/drops", dropper_app) -app.mount("/admin", admin_app) app.mount("/play", play_app) app.mount("/metatx", metatx_app) diff --git a/engineapi/engineapi/auth.py b/engineapi/engineapi/auth.py index 3fb05dd4..2b17b2d1 100644 --- a/engineapi/engineapi/auth.py +++ b/engineapi/engineapi/auth.py @@ -57,7 +57,6 @@ class MoonstreamAuthorization(EIP712Message): def sign_message(message_hash_bytes: HexBytes, private_key: HexBytes) -> HexBytes: - eth_private_key = eth_keys.keys.PrivateKey(private_key) _, _, _, signed_message_bytes = sign_message_hash( eth_private_key, message_hash_bytes diff --git a/engineapi/engineapi/contracts/web3_util.py b/engineapi/engineapi/contracts/web3_util.py index 34949230..f777da29 100644 --- a/engineapi/engineapi/contracts/web3_util.py +++ b/engineapi/engineapi/contracts/web3_util.py @@ -53,7 +53,6 @@ def get_nonce(web3: Web3, address: ChecksumAddress) -> Nonce: def submit_transaction( web3: Web3, transaction: Union[TxParams, Any], signer_private_key: str ) -> HexBytes: - """ Signs and submits json transaction to blockchain from the name of signer """ diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index b005ca9d..09064b93 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -1,9 +1,10 @@ from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set from uuid import UUID -from pydantic import BaseModel, Field, root_validator, validator +from bugout.data import BugoutResource +from pydantic import AnyHttpUrl, BaseModel, Field, root_validator, validator from web3 import Web3 @@ -23,6 +24,17 @@ class NowResponse(BaseModel): epoch_time: float +class CORSOrigins(BaseModel): + origins_set: Set[str] = Field(default_factory=set) + resources: List[BugoutResource] = Field(default_factory=list) + + +class IsCORSResponse(BaseModel): + origin: Optional[str] = None + updated_at: Optional[datetime] = None + created_at: Optional[datetime] = None + + class SignerListResponse(BaseModel): instances: List[Any] = Field(default_factory=list) diff --git a/engineapi/engineapi/middleware.py b/engineapi/engineapi/middleware.py index 9e4cc570..d43fcf34 100644 --- a/engineapi/engineapi/middleware.py +++ b/engineapi/engineapi/middleware.py @@ -1,20 +1,35 @@ import base64 import json import logging -from typing import Any, Awaitable, Callable, Dict, Optional +from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Set, Tuple +from uuid import UUID -from bugout.data import BugoutUser +from bugout.data import BugoutResource, BugoutResources, BugoutUser from bugout.exceptions import BugoutResponseException from fastapi import HTTPException, Request, Response +from pydantic import AnyHttpUrl, parse_obj_as from starlette.middleware.base import BaseHTTPMiddleware +from starlette.middleware.cors import CORSMiddleware +from starlette.responses import Response +from starlette.types import ASGIApp from web3 import Web3 +from . import data from .auth import ( MoonstreamAuthorizationExpired, MoonstreamAuthorizationVerificationError, verify, ) -from .settings import bugout_client as bc, MOONSTREAM_APPLICATION_ID +from .rc import REDIS_CONFIG_CORS_KEY, rc_client +from .settings import ( + ALLOW_ORIGINS, + BUGOUT_REQUEST_TIMEOUT_SECONDS, + BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_ADMIN_USER, + MOONSTREAM_APPLICATION_ID, +) +from .settings import bugout_client as bc logger = logging.getLogger(__name__) @@ -199,3 +214,198 @@ class ExtractBearerTokenMiddleware(BaseHTTPMiddleware): request.state.token = user_token return await call_next(request) + + +def parse_origins_from_resources( + resources: List[BugoutResources], +) -> data.CORSOrigins: + """ + Parse list of CORS origins with HTTP validation and remove duplications. + """ + cors_origins = data.CORSOrigins(origins_set=set()) + for resource in resources: + origin = resource.resource_data.get("origin", "") + try: + parse_obj_as(AnyHttpUrl, origin) + cors_origins.origins_set.add(origin) + cors_origins.resources.append(resource) + except Exception: + logger.warning( + f"Unable to parse origin: {origin} as URL from resource {resource.id}" + ) + continue + + return cors_origins + + +def check_default_origins(cors_origins: data.CORSOrigins) -> data.CORSOrigins: + """ + To prevent default origins loss. + """ + for o in ALLOW_ORIGINS: + if o not in cors_origins.origins_set: + cors_origins.origins_set.add(o) + return cors_origins + + +def create_application_settings_cors_origin( + token: str, user_id: Tuple[str, UUID], username: str, origin: str +) -> Optional[BugoutResource]: + resource: Optional[BugoutResource] = None + try: + resource = bc.create_resource( + token=token, + application_id=MOONSTREAM_APPLICATION_ID, + resource_data={ + "type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, + "setting": "cors", + "user_id": str(user_id), + "username": username, + "origin": origin, + }, + ) + if token != MOONSTREAM_ADMIN_ACCESS_TOKEN: + bc.add_resource_holder_permissions( + token=token, + resource_id=resource.id, + holder_permissions={ + "holder_id": str(MOONSTREAM_ADMIN_USER.id), + "holder_type": "user", + "permissions": ["admin", "create", "read", "update", "delete"], + }, + ) + except Exception as err: + logger.error( + f"Unable to write default CORS origin {origin} to Brood resource: {str(err)}" + ) + + return resource + + +def fetch_application_settings_cors_origins(token: str) -> data.CORSOrigins: + """ + Fetch application config resources with CORS origins setting. + If there are no such resources create new one with default origins from environment variable. + + Should return in any case some list of origins, by default it will be ALLOW_ORIGINS. + """ + + # Fetch CORS origins configs from resources for specified application + resources: BugoutResources + try: + resources = bc.list_resources( + token=token, + params={ + "application_id": MOONSTREAM_APPLICATION_ID, + "type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, + "setting": "cors", + }, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + + except Exception as err: + logger.error(f"Error fetching bugout resources with CORS origins: {str(err)}") + return data.CORSOrigins(origins_set=ALLOW_ORIGINS) + + # If there are no resources with CORS origins configuration, create resources + # for each default origin from environment variable + if len(resources.resources) == 0: + default_origins_cnt = 0 + for o in ALLOW_ORIGINS: + # Try to add new origins to Bugout resources application config, + # use 3 retries to assure origin added and not passed because of some network error. + retry_cnt = 0 + while retry_cnt < 3: + resource = create_application_settings_cors_origin( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + user_id=str(MOONSTREAM_ADMIN_USER.id), + username=MOONSTREAM_ADMIN_USER.username, + origin=o, + ) + if resource is not None: + resources.resources.append(resource) + default_origins_cnt += 1 + break + retry_cnt += 1 + + if default_origins_cnt != len(ALLOW_ORIGINS): + return data.CORSOrigins(origins_set=ALLOW_ORIGINS) + + logger.info( + f"Created resources with default {default_origins_cnt} CORS origins setting by moonstream admin user" + ) + + cors_origins: data.CORSOrigins = parse_origins_from_resources(resources.resources) + cors_origins = check_default_origins(cors_origins) + + return cors_origins + + +def set_cors_origins_cache(origins_set: Set[str]) -> None: + try: + rc_client.sadd(REDIS_CONFIG_CORS_KEY, *origins_set) + except Exception: + logger.warning("Unable to set CORS origins at Redis cache") + finally: + rc_client.close() + + +def fetch_and_set_cors_origins_cache() -> data.CORSOrigins: + cors_origins = fetch_application_settings_cors_origins( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN + ) + set_cors_origins_cache(cors_origins.origins_set) + + return cors_origins + + +class BugoutCORSMiddleware(CORSMiddleware): + """ + Modified CORSMiddleware from starlette.middleware.cors.py to work with Redis cache + and store application configuration for each user in Brood resources. + """ + + def __init__( + self, + app: ASGIApp, + allow_methods: Sequence[str] = ("GET",), + allow_headers: Sequence[str] = (), + allow_credentials: bool = False, + expose_headers: Sequence[str] = (), + max_age: int = 600, + ): + application_configs_allowed_origins: data.CORSOrigins = ( + fetch_and_set_cors_origins_cache() + ) + + super().__init__( + app=app, + allow_origins=list(application_configs_allowed_origins.origins_set), + allow_methods=allow_methods, + allow_headers=allow_headers, + allow_credentials=allow_credentials, + allow_origin_regex=None, + expose_headers=expose_headers, + max_age=max_age, + ) + + def is_allowed_origin(self, origin: str) -> bool: + if self.allow_all_origins: + return True + + if self.allow_origin_regex is not None and self.allow_origin_regex.fullmatch( + origin + ): + return True + + try: + is_allowed_origin = rc_client.sismember(REDIS_CONFIG_CORS_KEY, origin) + return is_allowed_origin + except Exception as err: + logger.warning( + f"Unable to fetch CORS origins from Redis cache, err: {str(err)}" + ) + finally: + rc_client.close() + + return origin in self.allow_origins diff --git a/engineapi/engineapi/rc.py b/engineapi/engineapi/rc.py new file mode 100644 index 00000000..e7961bf9 --- /dev/null +++ b/engineapi/engineapi/rc.py @@ -0,0 +1,43 @@ +from contextlib import asynccontextmanager + +from redis import ConnectionPool, Redis +from redis import asyncio as aioredis + +from .settings import ENGINE_REDIS_PASSWORD, ENGINE_REDIS_URL + +REDIS_CONFIG_CORS_KEY = "configs:cors:engineapi" + + +def create_redis_client() -> Redis: + rc_pool = ConnectionPool.from_url( + url=f"redis://:{ENGINE_REDIS_PASSWORD}@{ENGINE_REDIS_URL}", + max_connections=10, + decode_responses=True, + socket_timeout=0.5, + ) + return Redis(connection_pool=rc_pool) + + +rc_client = create_redis_client() + + +def create_async_redis_client() -> Redis: + rc_pool_async: ConnectionPool = aioredis.ConnectionPool.from_url( + url=f"redis://:{ENGINE_REDIS_PASSWORD}@{ENGINE_REDIS_URL}", + max_connections=10, + decode_responses=True, + socket_timeout=0.5, + ) + + return aioredis.Redis(connection_pool=rc_pool_async) + + +rc_client_async = create_async_redis_client() + + +@asynccontextmanager +async def yield_rc_async_session(): + try: + yield rc_client_async + finally: + await rc_client_async.close() diff --git a/engineapi/engineapi/routes/admin.py b/engineapi/engineapi/routes/admin.py index c9d2785e..a32ce4e5 100644 --- a/engineapi/engineapi/routes/admin.py +++ b/engineapi/engineapi/routes/admin.py @@ -7,15 +7,14 @@ from uuid import UUID from web3 import Web3 from fastapi import Body, FastAPI, Request, Depends, Query -from fastapi.middleware.cors import CORSMiddleware from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from .. import actions from .. import data from .. import db -from ..middleware import EngineHTTPException, EngineAuthMiddleware -from ..settings import DOCS_TARGET_PATH, ORIGINS +from ..middleware import EngineHTTPException, EngineAuthMiddleware, BugoutCORSMiddleware +from ..settings import DOCS_TARGET_PATH from ..version import VERSION @@ -46,8 +45,7 @@ app = FastAPI( app.add_middleware(EngineAuthMiddleware, whitelist=whitelist_paths) app.add_middleware( - CORSMiddleware, - allow_origins=ORIGINS, + BugoutCORSMiddleware, allow_credentials=True, allow_methods=["*"], allow_headers=["*"], @@ -113,7 +111,6 @@ async def create_drop( register_request: data.DropRegisterRequest = Body(...), db_session: Session = Depends(db.yield_db_session), ) -> data.DropCreatedResponse: - """ Create a drop for a given dropper contract. """ @@ -173,7 +170,6 @@ async def activate_drop( dropper_claim_id: UUID, db_session: Session = Depends(db.yield_db_session), ) -> data.DropUpdatedResponse: - """ Activate a given drop by drop id. """ @@ -220,7 +216,6 @@ async def deactivate_drop( dropper_claim_id: UUID, db_session: Session = Depends(db.yield_db_session), ) -> data.DropUpdatedResponse: - """ Activate a given drop by drop id. """ @@ -265,7 +260,6 @@ async def update_drop( update_request: data.DropUpdateRequest = Body(...), db_session: Session = Depends(db.yield_db_session), ) -> data.DropUpdatedResponse: - """ Update a given drop by drop id. """ @@ -407,7 +401,6 @@ async def delete_claimants( claimants_list: data.BatchRemoveClaimantsRequest = Body(...), db_session: Session = Depends(db.yield_db_session), ) -> data.RemoveClaimantsResponse: - """ Remove addresses to particular claim """ @@ -447,7 +440,6 @@ async def get_claimant_in_drop( address: str, db_session: Session = Depends(db.yield_db_session), ) -> data.Claimant: - """ Return claimant from drop """ diff --git a/engineapi/engineapi/routes/configs.py b/engineapi/engineapi/routes/configs.py new file mode 100644 index 00000000..351b4016 --- /dev/null +++ b/engineapi/engineapi/routes/configs.py @@ -0,0 +1,171 @@ +import logging +from typing import Any, Dict, List, Set + +from bugout.data import BugoutResource, BugoutResources +from fastapi import ( + BackgroundTasks, + Body, + Depends, + FastAPI, + Form, + HTTPException, + Query, + Request, +) +from pydantic import AnyHttpUrl + +from .. import data +from ..middleware import ( + BroodAuthMiddleware, + BugoutCORSMiddleware, + EngineHTTPException, + create_application_settings_cors_origin, + fetch_and_set_cors_origins_cache, + parse_origins_from_resources, +) +from ..settings import ( + BUGOUT_REQUEST_TIMEOUT_SECONDS, + BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, + DOCS_TARGET_PATH, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_ADMIN_USER, + MOONSTREAM_APPLICATION_ID, +) +from ..settings import bugout_client as bc +from ..version import VERSION + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +tags_metadata = [ + {"name": "configs", "description": "Moonstream Engine API configurations"} +] + +whitelist_paths: Dict[str, str] = {} +whitelist_paths.update( + { + "/configs/docs": "GET", + "/configs/openapi.json": "GET", + "/configs/is_origin": "GET", + } +) + +app = FastAPI( + title=f"Moonstream Engine API configurations", + description="Moonstream Engine API configurations endpoints.", + version=VERSION, + openapi_tags=tags_metadata, + openapi_url="/openapi.json", + docs_url=None, + redoc_url=f"/{DOCS_TARGET_PATH}", +) + + +app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths) + +app.add_middleware( + BugoutCORSMiddleware, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/is_origin", response_model=data.IsCORSResponse) +async def is_cors_origin(origin: str = Query(...)) -> data.IsCORSResponse: + is_cors_origin = data.IsCORSResponse() + try: + resources = bc.list_resources( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + params={ + "application_id": MOONSTREAM_APPLICATION_ID, + "type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, + "setting": "cors", + }, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + cors_origins: data.CORSOrigins = parse_origins_from_resources( + resources.resources + ) + if origin in cors_origins.origins_set: + for resource in cors_origins.resources: + resource_origin = resource.resource_data.get("origin", "") + # TODO(kompotkot): There are could be multiple creations by different users. + # Add logic to show most recent updated_at and oldest created_at. + if resource_origin == origin: + is_cors_origin.origin = resource_origin + is_cors_origin.created_at = resource.created_at + is_cors_origin.updated_at = resource.updated_at + except Exception as err: + logger.error(repr(err)) + raise EngineHTTPException(status_code=500) + + return is_cors_origin + + +@app.get("/origins", response_model=data.CORSOrigins) +async def get_cors_origins( + request: Request, +) -> data.CORSOrigins: + try: + resources = bc.list_resources( + token=request.state.token, + params={ + "application_id": MOONSTREAM_APPLICATION_ID, + "type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, + "setting": "cors", + }, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + cors_origins: data.CORSOrigins = parse_origins_from_resources( + resources.resources + ) + except Exception as err: + logger.error(repr(err)) + raise EngineHTTPException(status_code=500) + + return cors_origins + + +@app.post("/origin", response_model=data.CORSOrigins) +async def add_cors_origin( + request: Request, + background_tasks: BackgroundTasks, + new_origin: AnyHttpUrl = Form(...), +) -> data.CORSOrigins: + try: + resources = bc.list_resources( + token=request.state.token, + params={ + "application_id": MOONSTREAM_APPLICATION_ID, + "type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, + "setting": "cors", + }, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + ) + except Exception as err: + logger.error(f"Unable to fetch resource from Brood, err: {repr(err)}") + raise EngineHTTPException(status_code=500) + + cors_origins: data.CORSOrigins = parse_origins_from_resources(resources.resources) + + if new_origin in cors_origins.origins_set: + raise EngineHTTPException( + status_code=409, + detail=f"Provided origin {new_origin} already set by user", + ) + + resource = create_application_settings_cors_origin( + token=request.state.token, + user_id=request.state.user.id, + username=request.state.user.username, + origin=new_origin, + ) + cors_origins.origins_set.add(new_origin) + cors_origins.resources.append(resource) + + background_tasks.add_task( + fetch_and_set_cors_origins_cache, + ) + + return cors_origins diff --git a/engineapi/engineapi/routes/dropper.py b/engineapi/engineapi/routes/dropper.py index 33b97e54..8bf4486f 100644 --- a/engineapi/engineapi/routes/dropper.py +++ b/engineapi/engineapi/routes/dropper.py @@ -6,7 +6,6 @@ from typing import List, Optional, Any, Dict from uuid import UUID -from fastapi.middleware.cors import CORSMiddleware from fastapi import FastAPI, Body, Request, Depends, Query from hexbytes import HexBytes from sqlalchemy.orm import Session @@ -20,9 +19,8 @@ from ..contracts import Dropper_interface from .. import data from .. import db from .. import signatures -from ..middleware import EngineHTTPException, EngineAuthMiddleware +from ..middleware import EngineHTTPException, EngineAuthMiddleware, BugoutCORSMiddleware from ..settings import ( - ORIGINS, DOCS_TARGET_PATH, BLOCKCHAIN_WEB3_PROVIDERS, UNSUPPORTED_BLOCKCHAIN_ERROR_MESSAGE, @@ -65,8 +63,7 @@ app = FastAPI( app.add_middleware(EngineAuthMiddleware, whitelist=whitelist_paths) app.add_middleware( - CORSMiddleware, - allow_origins=ORIGINS, + BugoutCORSMiddleware, allow_credentials=True, allow_methods=["*"], allow_headers=["*"], @@ -220,11 +217,9 @@ async def get_drop_batch_handler( commit_required = False for claimant_drop in claimant_drops: - transformed_amount = claimant_drop.raw_amount if transformed_amount is None: - transformed_amount = actions.transform_claim_amount( db_session, claimant_drop.dropper_claim_id, claimant_drop.amount ) @@ -345,7 +340,6 @@ async def get_drops_terminus_handler( blockchain: str = Query(None), db_session: Session = Depends(db.yield_db_session), ) -> List[data.DropperTerminusResponse]: - """ Return distinct terminus pools """ @@ -512,7 +506,6 @@ async def create_drop( register_request: data.DropRegisterRequest = Body(...), db_session: Session = Depends(db.yield_db_session), ) -> data.DropCreatedResponse: - """ Create a drop for a given dropper contract. """ @@ -572,7 +565,6 @@ async def activate_drop( dropper_claim_id: UUID, db_session: Session = Depends(db.yield_db_session), ) -> data.DropUpdatedResponse: - """ Activate a given drop by drop id. """ @@ -619,7 +611,6 @@ async def deactivate_drop( dropper_claim_id: UUID, db_session: Session = Depends(db.yield_db_session), ) -> data.DropUpdatedResponse: - """ Activate a given drop by drop id. """ @@ -664,7 +655,6 @@ async def update_drop( update_request: data.DropUpdateRequest = Body(...), db_session: Session = Depends(db.yield_db_session), ) -> data.DropUpdatedResponse: - """ Update a given drop by drop id. """ @@ -795,7 +785,6 @@ async def delete_claimants( remove_claimants_request: data.DropRemoveClaimantsRequest = Body(...), db_session: Session = Depends(db.yield_db_session), ) -> data.RemoveClaimantsResponse: - """ Remove addresses to particular claim """ @@ -835,7 +824,6 @@ async def get_claimant_in_drop( address: str, db_session: Session = Depends(db.yield_db_session), ) -> data.Claimant: - """ Return claimant from drop """ diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index c0f9529f..4a690050 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -6,7 +6,6 @@ from uuid import UUID from web3 import Web3 from fastapi import FastAPI, Request, Depends, Response -from fastapi.middleware.cors import CORSMiddleware from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from typing import List, Optional @@ -14,7 +13,11 @@ from typing import List, Optional from .. import actions from .. import data from .. import db -from ..middleware import ExtractBearerTokenMiddleware, EngineHTTPException +from ..middleware import ( + ExtractBearerTokenMiddleware, + EngineHTTPException, + BugoutCORSMiddleware, +) from ..settings import DOCS_TARGET_PATH, bugout_client as bc from ..version import VERSION @@ -49,8 +52,7 @@ app = FastAPI( app.add_middleware(ExtractBearerTokenMiddleware, whitelist=leaderboad_whitelist) app.add_middleware( - CORSMiddleware, - allow_origins="*", + BugoutCORSMiddleware, allow_credentials=False, allow_methods=["*"], allow_headers=["*"], @@ -62,7 +64,6 @@ async def count_addresses( leaderboard_id: UUID, db_session: Session = Depends(db.yield_db_session), ): - """ Returns the number of addresses in the leaderboard. """ @@ -89,7 +90,6 @@ async def quartiles( leaderboard_id: UUID, db_session: Session = Depends(db.yield_db_session), ): - """ Returns the quartiles of the leaderboard. """ @@ -131,7 +131,6 @@ async def position( normalize_addresses: bool = True, db_session: Session = Depends(db.yield_db_session), ): - """ Returns the leaderboard posotion for the given address. With given window size. @@ -167,7 +166,6 @@ async def leaderboard( offset: int = 0, db_session: Session = Depends(db.yield_db_session), ) -> List[data.LeaderboardPosition]: - """ Returns the leaderboard positions. """ @@ -208,7 +206,6 @@ async def rank( offset: Optional[int] = None, db_session: Session = Depends(db.yield_db_session), ) -> List[data.LeaderboardPosition]: - """ Returns the leaderboard scores for the given rank. """ @@ -244,7 +241,6 @@ async def rank( async def ranks( leaderboard_id: UUID, db_session: Session = Depends(db.yield_db_session) ) -> List[data.RanksResponse]: - """ Returns the leaderboard rank buckets overview with score and size of bucket. """ @@ -282,7 +278,6 @@ async def leaderboard( normalize_addresses: bool = True, db_session: Session = Depends(db.yield_db_session), ): - """ Put the leaderboard to the database. """ diff --git a/engineapi/engineapi/routes/metatx.py b/engineapi/engineapi/routes/metatx.py index 3b0938ec..18bc6a7b 100644 --- a/engineapi/engineapi/routes/metatx.py +++ b/engineapi/engineapi/routes/metatx.py @@ -10,13 +10,12 @@ from typing import Dict, List, Optional from uuid import UUID from fastapi import Body, Depends, FastAPI, Query, Request, Path -from fastapi.middleware.cors import CORSMiddleware from sqlalchemy.exc import NoResultFound from sqlalchemy.orm import Session from .. import contracts_actions, data, db -from ..middleware import BroodAuthMiddleware, EngineHTTPException -from ..settings import DOCS_TARGET_PATH, ORIGINS +from ..middleware import BroodAuthMiddleware, EngineHTTPException, BugoutCORSMiddleware +from ..settings import DOCS_TARGET_PATH from ..version import VERSION logger = logging.getLogger(__name__) @@ -56,8 +55,7 @@ app = FastAPI( app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths) app.add_middleware( - CORSMiddleware, - allow_origins=ORIGINS, + BugoutCORSMiddleware, allow_credentials=True, allow_methods=["*"], allow_headers=["*"], diff --git a/engineapi/engineapi/routes/play.py b/engineapi/engineapi/routes/play.py index 20c36128..01f5c4e9 100644 --- a/engineapi/engineapi/routes/play.py +++ b/engineapi/engineapi/routes/play.py @@ -10,7 +10,6 @@ from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from hexbytes import HexBytes from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware from web3 import Web3 from ..models import DropperClaimant @@ -19,7 +18,7 @@ from .. import data from .. import db from .. import signatures from ..contracts import Dropper_interface -from ..middleware import EngineHTTPException +from ..middleware import EngineHTTPException, BugoutCORSMiddleware from ..settings import BLOCKCHAIN_WEB3_PROVIDERS, DOCS_TARGET_PATH from ..version import VERSION @@ -42,8 +41,7 @@ app = FastAPI( app.add_middleware( - CORSMiddleware, - allow_origins="*", + BugoutCORSMiddleware, allow_credentials=False, allow_methods=["*"], allow_headers=["*"], @@ -126,11 +124,9 @@ async def get_drop_batch_handler( commit_required = False for claimant_drop in claimant_drops: - transformed_amount = claimant_drop.raw_amount if transformed_amount is None: - transformed_amount = actions.transform_claim_amount( db_session, claimant_drop.dropper_claim_id, claimant_drop.amount ) @@ -394,7 +390,6 @@ async def get_drops_terminus_handler( blockchain: str = Query(None), db_session: Session = Depends(db.yield_db_session), ) -> List[data.DropperTerminusResponse]: - """ Return distinct terminus pools """ diff --git a/engineapi/engineapi/scripts/fill_raw_amount.py b/engineapi/engineapi/scripts/fill_raw_amount.py index a86eeadc..b8250c53 100644 --- a/engineapi/engineapi/scripts/fill_raw_amount.py +++ b/engineapi/engineapi/scripts/fill_raw_amount.py @@ -16,7 +16,6 @@ def run_fill_raw_amount(args: argparse.Namespace): token_types: Dict[str, Dict[str, List[Dict[str, Any]]]] = dict() with db.yield_db_session_ctx() as db_session: - res = db_session.execute( """select distinct dropper_contracts.blockchain, dropper_contracts.address, dropper_claims.claim_id from dropper_contracts left join dropper_claims on dropper_contracts.id = dropper_claims.dropper_contract_id diff --git a/engineapi/engineapi/settings.py b/engineapi/engineapi/settings.py index be2de443..90780c6f 100644 --- a/engineapi/engineapi/settings.py +++ b/engineapi/engineapi/settings.py @@ -1,9 +1,14 @@ +import logging import os import warnings +from typing import Optional, Set -from web3 import Web3, HTTPProvider -from web3.middleware import geth_poa_middleware from bugout.app import Bugout +from bugout.data import BugoutUser +from web3 import HTTPProvider, Web3 +from web3.middleware import geth_poa_middleware + +logger = logging.getLogger(__name__) # Bugout BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev") @@ -21,7 +26,17 @@ if RAW_ORIGINS is None: raise ValueError( "ENGINE_CORS_ALLOWED_ORIGINS environment variable must be set (comma-separated list of CORS allowed origins)" ) -ORIGINS = RAW_ORIGINS.split(",") +RAW_ORIGINS_LST = RAW_ORIGINS.split(",") +ALLOW_ORIGINS: Set[str] = set() +for o_raw in RAW_ORIGINS_LST: + ALLOW_ORIGINS.add(o_raw.strip()) + + +BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG = "application-config" +BUGOUT_REQUEST_TIMEOUT_SECONDS = 5 + +ENGINE_REDIS_URL = os.environ.get("ENGINE_REDIS_URL") +ENGINE_REDIS_PASSWORD = os.environ.get("ENGINE_REDIS_PASSWORD") # Open API documentation path DOCS_TARGET_PATH = os.environ.get("DOCS_TARGET_PATH", "docs") @@ -178,3 +193,12 @@ LEADERBOARD_RESOURCE_TYPE = "leaderboard" MOONSTREAM_ADMIN_ACCESS_TOKEN = os.environ.get("MOONSTREAM_ADMIN_ACCESS_TOKEN", "") if MOONSTREAM_ADMIN_ACCESS_TOKEN == "": raise ValueError("MOONSTREAM_ADMIN_ACCESS_TOKEN environment variable must be set") + +MOONSTREAM_ADMIN_USER: Optional[BugoutUser] = None +try: + MOONSTREAM_ADMIN_USER = bugout_client.get_user( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + ) +except Exception as err: + logger.error(f"Unable to get Moonstream admin user with token, err: {str(err)}") + logger.error("Running application partly functional") diff --git a/engineapi/engineapi/signatures.py b/engineapi/engineapi/signatures.py index b9a19c19..66c86ae3 100644 --- a/engineapi/engineapi/signatures.py +++ b/engineapi/engineapi/signatures.py @@ -98,7 +98,6 @@ class AccountSigner(Signer): return signed_message_bytes.hex() def batch_sign_message(self, messages_list: List[str]): - signed_messages_list = {} for message in messages_list: diff --git a/engineapi/engineapi/test_middleware.py b/engineapi/engineapi/test_middleware.py new file mode 100644 index 00000000..7483fd5c --- /dev/null +++ b/engineapi/engineapi/test_middleware.py @@ -0,0 +1,43 @@ +import unittest +import uuid +from datetime import datetime + +from bugout.data import BugoutResource, BugoutResources, BugoutUser +from pydantic import AnyHttpUrl, parse_obj_as + +from .middleware import parse_origins_from_resources +from .settings import BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG + +TEST_ALLOW_ORIGINS = ["http://localhost:3000", "http://localhost:4000", "wrong one"] + + +class TestInit(unittest.TestCase): + def setUp(self): + utc_now = datetime.utcnow() + self.resources: BugoutResources = BugoutResources( + resources=[ + BugoutResource( + id=uuid.uuid4(), + application_id=str(uuid.uuid4()), + resource_data={ + "type": BUGOUT_RESOURCE_TYPE_APPLICATION_CONFIG, + "setting": "cors", + "user_id": str(uuid.uuid4()), + "cors": TEST_ALLOW_ORIGINS, + }, + created_at=utc_now, + updated_at=utc_now, + ) + ] + ) + + def test_parse_origins_from_resources(self): + cnt = 0 + for o in TEST_ALLOW_ORIGINS: + try: + parse_obj_as(AnyHttpUrl, o) + cnt += 1 + except Exception: + continue + cors_origins = parse_origins_from_resources(self.resources) + self.assertEqual(cnt, len(cors_origins)) diff --git a/engineapi/sample.env b/engineapi/sample.env index 78a4b668..d910d071 100644 --- a/engineapi/sample.env +++ b/engineapi/sample.env @@ -9,6 +9,8 @@ export ENGINE_DB_URI="postgresql://:@:/=2.3.0", "fastapi", + "redis", "psycopg2-binary", "pydantic", + "python-multipart", "sqlalchemy", "tqdm", "uvicorn",