From fa4baa9ba843cb466f05ff310e8894f0515101dc Mon Sep 17 00:00:00 2001 From: Andrey Dolgolev Date: Thu, 13 Jul 2023 15:16:15 +0300 Subject: [PATCH 1/3] Revert "Revert "Add tags and descriptions"" --- moonstreamapi/moonstreamapi/data.py | 43 ++++++- .../moonstreamapi/routes/subscriptions.py | 120 ++++++++++++++---- moonstreamapi/moonstreamapi/settings.py | 10 ++ 3 files changed, 143 insertions(+), 30 deletions(-) diff --git a/moonstreamapi/moonstreamapi/data.py b/moonstreamapi/moonstreamapi/data.py index e0e12751..150c6c16 100644 --- a/moonstreamapi/moonstreamapi/data.py +++ b/moonstreamapi/moonstreamapi/data.py @@ -2,14 +2,17 @@ Pydantic schemas for the Moonstream HTTP API """ from datetime import datetime +import json from enum import Enum from typing import Any, Dict, List, Optional, Union, Literal from uuid import UUID from xmlrpc.client import Boolean +from fastapi import Form from pydantic import BaseModel, Field, validator from sqlalchemy import false + USER_ONBOARDING_STATE = "onboarding_state" BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" @@ -47,6 +50,8 @@ class SubscriptionResourceData(BaseModel): abi: Optional[str] color: Optional[str] label: Optional[str] + description: Optional[str] = None + tags: List[Dict[str, Any]] = Field(default_factory=list) user_id: str subscription_type_id: Optional[str] created_at: Optional[datetime] @@ -243,6 +248,40 @@ class SubdcriptionsAbiResponse(BaseModel): abi: str +class UpdateSubscriptionRequest(BaseModel): + color: Optional[str] = Form(None) + label: Optional[str] = Form(None) + abi: Optional[str] = Form(None) + description: Optional[str] = Form(None) + tags: Optional[List[Dict[str, str]]] = Form(None) + + @validator("tags", pre=True, always=True) + def transform_to_dict(cls, v): + if isinstance(v, str): + return json.loads(v) + elif isinstance(v, list): + return v + return [] + + +class CreateSubscriptionRequest(BaseModel): + address: str = Form(...) + subscription_type_id: str = Form(...) + color: str = Form(...) + label: str = Form(...) + abi: Optional[str] = Form(None) + description: Optional[str] = Form(None) + tags: Optional[List[Dict[str, str]]] = Form(None) + + @validator("tags", pre=True, always=True) + def transform_to_dict(cls, v): + if isinstance(v, str): + return json.loads(v) + elif isinstance(v, list): + return v + return [] + + class DashboardMeta(BaseModel): subscription_id: UUID generic: Optional[List[Dict[str, str]]] @@ -295,8 +334,8 @@ class QueryInfoResponse(BaseModel): preapprove: bool = False approved: bool = False parameters: Dict[str, Any] = Field(default_factory=dict) - created_at: Optional[datetime] - updated_at: Optional[datetime] + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None class SuggestedQueriesResponse(BaseModel): diff --git a/moonstreamapi/moonstreamapi/routes/subscriptions.py b/moonstreamapi/moonstreamapi/routes/subscriptions.py index 9722ca88..dbb2f556 100644 --- a/moonstreamapi/moonstreamapi/routes/subscriptions.py +++ b/moonstreamapi/moonstreamapi/routes/subscriptions.py @@ -6,7 +6,6 @@ import hashlib import json import logging from typing import Any, Dict, List, Optional -import traceback from bugout.exceptions import BugoutResponseException from bugout.data import BugoutSearchResult @@ -29,7 +28,11 @@ from ..admin import subscription_types from ..middleware import MoonstreamHTTPException from ..reporter import reporter from ..settings import bugout_client as bc, entity_client as ec -from ..settings import MOONSTREAM_ADMIN_ACCESS_TOKEN, THREAD_TIMEOUT_SECONDS +from ..settings import ( + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_ENTITIES_RESERVED_TAGS, + THREAD_TIMEOUT_SECONDS, +) from ..web3_provider import ( yield_web3_provider, ) @@ -49,11 +52,6 @@ BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION = "entity_subscription" async def add_subscription_handler( request: Request, background_tasks: BackgroundTasks, - address: str = Form(...), - color: str = Form(...), - label: str = Form(...), - subscription_type_id: str = Form(...), - abi: Optional[str] = Form(None), web3: Web3 = Depends(yield_web3_provider), ) -> data.SubscriptionResourceData: """ @@ -61,6 +59,21 @@ async def add_subscription_handler( """ token = request.state.token + form = await request.form() + + try: + form_data = data.UpdateSubscriptionRequest(**form) + except Exception as e: + raise MoonstreamHTTPException(status_code=400, detail=str(e)) + + address = form_data.address + color = form_data.color + label = form_data.label + abi = form_data.abi + description = form_data.description + tags = form_data.tags + subscription_type_id = form_data.subscription_type_id + if subscription_type_id != "ethereum_whalewatch": try: address = web3.toChecksumAddress(address) @@ -124,6 +137,28 @@ async def add_subscription_handler( address, ) + if description: + content["description"] = description + + allowed_required_fields = [] + if tags: + allowed_required_fields = [ + item + for item in tags + if not any(key in item for key in MOONSTREAM_ENTITIES_RESERVED_TAGS) + ] + + required_fields = [ + {"type": "subscription"}, + {"subscription_type_id": f"{subscription_type_id}"}, + {"color": f"{color}"}, + {"label": f"{label}"}, + {"user_id": f"{user.id}"}, + ] + + if allowed_required_fields: + required_fields.extend(allowed_required_fields) + try: collection_id = get_entity_subscription_collection_id( resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, @@ -140,13 +175,7 @@ async def add_subscription_handler( subscription_type_id ].blockchain, name=label, - required_fields=[ - {"type": "subscription"}, - {"subscription_type_id": f"{subscription_type_id}"}, - {"color": f"{color}"}, - {"label": f"{label}"}, - {"user_id": f"{user.id}"}, - ], + required_fields=required_fields, secondary_fields=content, ) except EntityCollectionNotFoundException as e: @@ -170,6 +199,8 @@ async def add_subscription_handler( color=color, label=label, abi=entity.secondary_fields.get("abi"), + description=entity.secondary_fields.get("description"), + tags=entity.required_fields, subscription_type_id=subscription_type_id, updated_at=entity.updated_at, created_at=entity.created_at, @@ -240,6 +271,8 @@ async def delete_subscription_handler(request: Request, subscription_id: str): color=color, label=label, abi=abi, + description=deleted_entity.secondary_fields.get("description"), + tags=deleted_entity.required_fields, subscription_type_id=subscription_type_id, updated_at=deleted_entity.updated_at, created_at=deleted_entity.created_at, @@ -311,6 +344,8 @@ async def get_subscriptions_handler( color=color, label=label, abi="True" if subscription.secondary_fields.get("abi") else None, + description=subscription.secondary_fields.get("description"), + tags=subscription.required_fields, subscription_type_id=subscription_type_id, updated_at=subscription.updated_at, created_at=subscription.created_at, @@ -328,9 +363,6 @@ async def update_subscriptions_handler( request: Request, subscription_id: str, background_tasks: BackgroundTasks, - color: Optional[str] = Form(None), - label: Optional[str] = Form(None), - abi: Optional[str] = Form(None), ) -> data.SubscriptionResourceData: """ Get user's subscriptions. @@ -339,9 +371,17 @@ async def update_subscriptions_handler( user = request.state.user - update_required_fields = [] + form = await request.form() + try: + form_data = data.UpdateSubscriptionRequest(**form) + except Exception as e: + raise MoonstreamHTTPException(status_code=400, detail=str(e)) - update_secondary_fields = {} + color = form_data.color + label = form_data.label + abi = form_data.abi + description = form_data.description + tags = form_data.tags try: collection_id = get_entity_subscription_collection_id( @@ -359,7 +399,13 @@ async def update_subscriptions_handler( subscription_type_id = None - update_required_fields = subscription_entity.required_fields + update_required_fields = [ + field + for field in subscription_entity.required_fields + if any(key in field for key in MOONSTREAM_ENTITIES_RESERVED_TAGS) + ] + + update_secondary_fields = subscription_entity.secondary_fields for field in update_required_fields: if "subscription_type_id" in field: @@ -370,7 +416,7 @@ async def update_subscriptions_handler( f"Subscription entity {subscription_id} in collection {collection_id} has no subscription_type_id malformed subscription entity" ) raise MoonstreamHTTPException( - status_code=404, + status_code=409, detail="Not valid subscription entity", ) @@ -387,13 +433,19 @@ async def update_subscriptions_handler( raise MoonstreamHTTPException(status_code=500, internal_error=e) for field in update_required_fields: - if "color" in field and color is not None: - field["color"] = color + if "color" in field: + if color is not None: + field["color"] = color + else: + color = field["color"] - if "label" in field and label is not None: - field["label"] = label + if "label" in field: + if label is not None: + field["label"] = label + else: + label = field["label"] - if abi: + if abi is not None: try: json_abi = json.loads(abi) except json.JSONDecodeError: @@ -407,9 +459,19 @@ async def update_subscriptions_handler( hash = hashlib.md5(abi_string.encode("utf-8")).hexdigest() update_secondary_fields["abi_hash"] = hash - else: - update_secondary_fields = subscription_entity.secondary_fields + if description is not None: + update_secondary_fields["description"] = description + + if tags: + allowed_required_fields = [ + item + for item in tags + if not any(key in item for key in MOONSTREAM_ENTITIES_RESERVED_TAGS) + ] + + if allowed_required_fields: + update_required_fields.extend(allowed_required_fields) try: subscription = ec.update_entity( token=token, @@ -441,6 +503,8 @@ async def update_subscriptions_handler( color=color, label=label, abi=subscription.secondary_fields.get("abi"), + description=subscription.secondary_fields.get("description"), + tags=subscription.required_fields, subscription_type_id=subscription_type_id, updated_at=subscription_entity.updated_at, created_at=subscription_entity.created_at, diff --git a/moonstreamapi/moonstreamapi/settings.py b/moonstreamapi/moonstreamapi/settings.py index 1a14503c..33589f4b 100644 --- a/moonstreamapi/moonstreamapi/settings.py +++ b/moonstreamapi/moonstreamapi/settings.py @@ -150,6 +150,16 @@ if MOONSTREAM_S3_QUERIES_BUCKET_PREFIX == "": "MOONSTREAM_S3_QUERIES_BUCKET_PREFIX environment variable must be set" ) +# Entities reserved tags +MOONSTREAM_ENTITIES_RESERVED_TAGS = [ + "type", + "subscription_type_id", + "color", + "label", + "user_id", + "address", + "blockchain", +] ## Moonstream resources types From 91ec48d7ac556faff7135c384096437ce6210ad2 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 13 Jul 2023 09:37:23 +0300 Subject: [PATCH 2/3] Add changes. --- moonstreamapi/moonstreamapi/data.py | 2 +- .../moonstreamapi/routes/subscriptions.py | 27 ++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/moonstreamapi/moonstreamapi/data.py b/moonstreamapi/moonstreamapi/data.py index 150c6c16..9ea4cbbb 100644 --- a/moonstreamapi/moonstreamapi/data.py +++ b/moonstreamapi/moonstreamapi/data.py @@ -51,7 +51,7 @@ class SubscriptionResourceData(BaseModel): color: Optional[str] label: Optional[str] description: Optional[str] = None - tags: List[Dict[str, Any]] = Field(default_factory=list) + tags: List[str] = Field(default_factory=list) user_id: str subscription_type_id: Optional[str] created_at: Optional[datetime] diff --git a/moonstreamapi/moonstreamapi/routes/subscriptions.py b/moonstreamapi/moonstreamapi/routes/subscriptions.py index dbb2f556..01700265 100644 --- a/moonstreamapi/moonstreamapi/routes/subscriptions.py +++ b/moonstreamapi/moonstreamapi/routes/subscriptions.py @@ -192,6 +192,13 @@ async def add_subscription_handler( detail="Currently unable to get collection id", ) + normalized_entity_tags = [ + f"{key}:{value}" + for tag in entity.required_fields + for key, value in tag.items() + if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS + ] + return data.SubscriptionResourceData( id=str(entity.entity_id), user_id=str(user.id), @@ -200,7 +207,7 @@ async def add_subscription_handler( label=label, abi=entity.secondary_fields.get("abi"), description=entity.secondary_fields.get("description"), - tags=entity.required_fields, + tags=normalized_entity_tags, subscription_type_id=subscription_type_id, updated_at=entity.updated_at, created_at=entity.created_at, @@ -336,6 +343,13 @@ async def get_subscriptions_handler( if "label" in tag: label = tag["label"] + normalized_entity_tags = [ + f"{key}:{value}" + for tag in tags + for key, value in tag.items() + if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS + ] + subscriptions.append( data.SubscriptionResourceData( id=str(subscription.entity_id), @@ -345,7 +359,7 @@ async def get_subscriptions_handler( label=label, abi="True" if subscription.secondary_fields.get("abi") else None, description=subscription.secondary_fields.get("description"), - tags=subscription.required_fields, + tags=normalized_entity_tags, subscription_type_id=subscription_type_id, updated_at=subscription.updated_at, created_at=subscription.created_at, @@ -496,6 +510,13 @@ async def update_subscriptions_handler( subscription.address, ) + normalized_entity_tags = [ + f"{key}:{value}" + for tag in subscription.required_fields + for key, value in tag.items() + if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS + ] + return data.SubscriptionResourceData( id=str(subscription.entity_id), user_id=str(user.id), @@ -504,7 +525,7 @@ async def update_subscriptions_handler( label=label, abi=subscription.secondary_fields.get("abi"), description=subscription.secondary_fields.get("description"), - tags=subscription.required_fields, + tags=normalized_entity_tags, subscription_type_id=subscription_type_id, updated_at=subscription_entity.updated_at, created_at=subscription_entity.created_at, From 9d577b676e6aac865f5a592d508528ea6735194a Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 13 Jul 2023 09:52:36 +0300 Subject: [PATCH 3/3] Add changes. --- moonstreamapi/moonstreamapi/routes/subscriptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moonstreamapi/moonstreamapi/routes/subscriptions.py b/moonstreamapi/moonstreamapi/routes/subscriptions.py index 01700265..4a8722c4 100644 --- a/moonstreamapi/moonstreamapi/routes/subscriptions.py +++ b/moonstreamapi/moonstreamapi/routes/subscriptions.py @@ -62,7 +62,7 @@ async def add_subscription_handler( form = await request.form() try: - form_data = data.UpdateSubscriptionRequest(**form) + form_data = data.CreateSubscriptionRequest(**form) except Exception as e: raise MoonstreamHTTPException(status_code=400, detail=str(e))