From 323784f8b2cd0b06a7f75b268542d2345486a6d0 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 27 Jul 2023 09:22:23 +0300 Subject: [PATCH 01/50] Fix function filter degradetion. --- crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py index 85a3f078..04fc2ed9 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py @@ -189,7 +189,11 @@ def handle_historical_crawl(args: argparse.Namespace) -> None: ) if addresses_filter: - filtered_function_call_jobs = [job for job in all_function_call_jobs] + filtered_function_call_jobs = [ + job + for job in all_function_call_jobs + if job.contract_address in addresses_filter + ] else: filtered_function_call_jobs = all_function_call_jobs @@ -267,6 +271,8 @@ def handle_historical_crawl(args: argparse.Namespace) -> None: end_block = args.end + start_block = args.start + # get set of addresses from event jobs and function call jobs if args.find_deployed_blocks: addresses_set = set() From 6462d84d6514545511c4bcdf0fc3e2013db4f74e Mon Sep 17 00:00:00 2001 From: Andrey Date: Fri, 28 Jul 2023 08:28:14 +0300 Subject: [PATCH 02/50] Add missing parameter. --- crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py index f20fe185..fdffa077 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -109,7 +109,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: logger.error(f"Could not get results for query {query_name} in time") continue - leaderboard_push_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/scores?normalize_addresses={leaderboard_data['normalize_addresses']}" + leaderboard_push_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/scores?normalize_addresses={leaderboard_data['normalize_addresses']}&overwrite=true" leaderboard_api_headers = { "Authorization": f"Bearer {args.query_api_access_token}", From a3b0841d10c0068e9fd7883c750f4a1618dd183e Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 31 Jul 2023 12:32:19 +0000 Subject: [PATCH 03/50] Moonstream API migration to spire entity --- moonstreamapi/moonstreamapi/actions.py | 92 ++++---- .../generate_entity_subscriptions.py | 205 +++++++++--------- moonstreamapi/moonstreamapi/admin/queries.py | 14 +- moonstreamapi/moonstreamapi/data.py | 7 +- .../moonstreamapi/routes/dashboards.py | 45 ++-- .../moonstreamapi/routes/subscriptions.py | 126 +++++------ moonstreamapi/moonstreamapi/routes/users.py | 2 +- moonstreamapi/requirements.txt | 2 +- moonstreamapi/setup.py | 4 +- 9 files changed, 236 insertions(+), 261 deletions(-) diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index a4360e94..7c908126 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -1,28 +1,27 @@ -from collections import OrderedDict import hashlib import json -from itertools import chain import logging -from typing import List, Optional, Dict, Any, Union -from enum import Enum import uuid +from collections import OrderedDict +from enum import Enum +from itertools import chain +from typing import Any, Dict, List, Optional, Union import boto3 # type: ignore - from bugout.data import ( - BugoutSearchResults, - BugoutSearchResult, + BugoutJournal, + BugoutJournals, BugoutResource, BugoutResources, + BugoutSearchResult, + BugoutSearchResults, ) -from bugout.journal import SearchOrder from bugout.exceptions import BugoutResponseException -from entity.data import EntityCollectionsResponse, EntityCollectionResponse # type: ignore -from entity.exceptions import EntityUnexpectedResponse # type: ignore +from bugout.journal import SearchOrder from ens.utils import is_valid_ens_name # type: ignore from eth_utils.address import is_address # type: ignore -from moonstreamdb.models import EthereumLabel from moonstreamdb.blockchain import AvailableBlockchainType +from moonstreamdb.models import EthereumLabel from slugify import slugify # type: ignore from sqlalchemy import text from sqlalchemy.orm import Session @@ -32,24 +31,20 @@ from web3._utils.validation import validate_abi from . import data from .middleware import MoonstreamHTTPException from .reporter import reporter +from .selectors_storage import selectors from .settings import ( BUGOUT_REQUEST_TIMEOUT_SECONDS, ETHERSCAN_SMARTCONTRACTS_BUCKET, MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_APPLICATION_ID, MOONSTREAM_DATA_JOURNAL_ID, + MOONSTREAM_MOONWORM_TASKS_JOURNAL, MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, - MOONSTREAM_MOONWORM_TASKS_JOURNAL, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - support_interfaces, - supportsInterface_abi, - multicall_contracts, ) -from .settings import bugout_client as bc, entity_client as ec -from .web3_provider import multicall, FunctionSignature, connect -from .selectors_storage import selectors - +from .settings import bugout_client as bc +from .settings import multicall_contracts, support_interfaces, supportsInterface_abi +from .web3_provider import FunctionSignature, connect, multicall logger = logging.getLogger(__name__) @@ -88,9 +83,9 @@ class ResourceQueryFetchException(Exception): """ -class EntityCollectionNotFoundException(Exception): +class EntityJournalNotFoundException(Exception): """ - Raised when entity collection is not found + Raised when journal (collection prev.) with entities not found. """ @@ -641,14 +636,14 @@ def get_query_by_name(query_name: str, token: uuid.UUID) -> str: return query_id -def get_entity_subscription_collection_id( +def get_entity_subscription_journal_id( resource_type: str, token: Union[uuid.UUID, str], user_id: uuid.UUID, create_if_not_exist: bool = False, ) -> Optional[str]: """ - Get collection_id from brood resources. If collection not exist and create_if_not_exist is True + Get collection_id (journal_id) from brood resources. If journal not exist and create_if_not_exist is True """ params = { @@ -668,52 +663,49 @@ def get_entity_subscription_collection_id( if len(resources.resources) == 0: if not create_if_not_exist: - raise EntityCollectionNotFoundException( - "Subscription collection not found." - ) - collection_id = generate_collection_for_user(resource_type, token, user_id) + raise EntityJournalNotFoundException("Subscription journal not found.") + journal_id = generate_journal_for_user(resource_type, token, user_id) - return collection_id + return journal_id else: resource = resources.resources[0] return resource.resource_data["collection_id"] -def generate_collection_for_user( +def generate_journal_for_user( resource_type: str, token: Union[uuid.UUID, str], user_id: uuid.UUID, ) -> str: try: - # try get collection + # Try get journal - collections: EntityCollectionsResponse = ec.list_collections(token=token) + journals: BugoutJournals = bc.list_journals(token=token) - available_collections: Dict[str, str] = { - collection.name: collection.collection_id - for collection in collections.collections + available_journals: Dict[str, str] = { + journal.name: journal.id for journal in journals.journals } - subscription_collection_name = f"subscriptions_{user_id}" + subscription_journal_name = f"subscriptions_{user_id}" - if subscription_collection_name not in available_collections: - collection: EntityCollectionResponse = ec.add_collection( - token=token, name=subscription_collection_name + if subscription_journal_name not in available_journals: + journal: BugoutJournal = bc.create_journal( + token=token, name=subscription_journal_name ) - collection_id = collection.collection_id + journal_id = journal.id else: - collection_id = available_collections[subscription_collection_name] - except EntityUnexpectedResponse as e: - logger.error(f"Error create collection, error: {str(e)}") + journal_id = available_journals[subscription_journal_name] + except Exception as e: + logger.error(f"Error create journal, error: {str(e)}") raise MoonstreamHTTPException( - status_code=500, detail="Can't create collection for subscriptions" + status_code=500, detail="Can't create journal for subscriptions" ) resource_data = { "type": resource_type, "user_id": str(user_id), - "collection_id": str(collection_id), + "collection_id": str(journal_id), } try: @@ -727,14 +719,14 @@ def generate_collection_for_user( except Exception as e: logger.error(f"Error creating subscription resource: {str(e)}") logger.error( - f"Required create resource data: {resource_data}, and grand access to journal: {collection_id}, for user: {user_id}" + f"Required create resource data: {resource_data}, and grand access to journal: {journal_id}, for user: {user_id}" ) raise MoonstreamHTTPException(status_code=500, internal_error=e) try: bc.update_journal_scopes( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - journal_id=collection_id, + journal_id=journal_id, holder_type="user", holder_id=user_id, permission_list=[ @@ -746,16 +738,16 @@ def generate_collection_for_user( ], ) logger.info( - f"Grand access to journal: {collection_id}, for user: {user_id} successfully" + f"Grand access to journal: {journal_id}, for user: {user_id} successfully" ) except Exception as e: logger.error(f"Error updating journal scopes: {str(e)}") logger.error( - f"Required grand access to journal: {collection_id}, for user: {user_id}" + f"Required grand access to journal: {journal_id}, for user: {user_id}" ) raise MoonstreamHTTPException(status_code=500, internal_error=e) - return collection_id + return journal_id def generate_s3_access_links( diff --git a/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py b/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py index 5b73c7bc..c63459d8 100644 --- a/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py +++ b/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py @@ -2,65 +2,65 @@ Generate entity subscriptions from existing brood resources subscriptions """ import hashlib -import logging import json +import logging import os import traceback -from typing import List, Optional, Dict, Any, Union, Tuple import uuid -import time +from typing import Any, Dict, List, Optional, Tuple, Union import boto3 # type: ignore -from bugout.data import BugoutResources, BugoutResource -from bugout.exceptions import BugoutResponseException -from entity.exceptions import EntityUnexpectedResponse # type: ignore -from entity.data import EntityCollectionResponse, EntityResponse # type: ignore +from bugout.data import ( + BugoutJournal, + BugoutJournalEntity, + BugoutResource, + BugoutResources, +) +from bugout.exceptions import BugoutResponseException, BugoutUnexpectedResponse from ...settings import ( BUGOUT_REQUEST_TIMEOUT_SECONDS, - MOONSTREAM_ADMIN_ACCESS_TOKEN, - BUGOUT_RESOURCE_TYPE_SUBSCRIPTION, - BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, - MOONSTREAM_APPLICATION_ID, BUGOUT_RESOURCE_TYPE_DASHBOARD, + BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, + BUGOUT_RESOURCE_TYPE_SUBSCRIPTION, + MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_APPLICATION_ID, ) -from ...settings import bugout_client as bc, entity_client as ec +from ...settings import bugout_client as bc from ..subscription_types import CANONICAL_SUBSCRIPTION_TYPES logger = logging.getLogger(__name__) -### create collection for user +### Create journal for user -def create_collection_for_user(user_id: uuid.UUID) -> str: +def create_journal_for_user(user_id: uuid.UUID) -> str: """ - Create collection for user if not exist + Create journal (collection) for user if not exist """ try: - # try get collection - - collection: EntityCollectionResponse = ec.add_collection( + # Try to get journal + journal: BugoutJournal = bc.create_journal( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, name=f"subscriptions_{user_id}" ) - collection_id = collection.collection_id - - except EntityUnexpectedResponse as e: - logger.error(f"Error create collection, error: {str(e)}") - return str(collection_id) + journal_id = journal.id + except BugoutUnexpectedResponse as e: + logger.error(f"Error create journal, error: {str(e)}") + return str(journal_id) def add_entity_subscription( user_id: uuid.UUID, subscription_type_id: str, - collection_id: str, + journal_id: str, address: str, color: str, label: str, content: Dict[str, Any], -) -> EntityResponse: +) -> BugoutJournalEntity: """ - Add subscription to collection + Add subscription to journal (collection). """ if subscription_type_id not in CANONICAL_SUBSCRIPTION_TYPES: @@ -73,12 +73,12 @@ def add_entity_subscription( f"Subscription type ID {subscription_type_id} is not a blockchain subscription type." ) - entity = ec.add_entity( + entity = bc.create_entity( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - collection_id=collection_id, + journal_id=journal_id, address=address, blockchain=CANONICAL_SUBSCRIPTION_TYPES[subscription_type_id].blockchain, - name=label, + title=label, required_fields=[ {"type": "subscription"}, {"subscription_type_id": f"{subscription_type_id}"}, @@ -105,34 +105,19 @@ def get_abi_from_s3(s3_path: str, bucket: str): logger.error(f"Error get ABI from S3: {str(e)}") -def revoke_collection_permissions_from_user( - user_id: uuid.UUID, collection_id: str, permissions: List[str] -): - """ - Remove all permissions from user - """ - bc.delete_journal_scopes( - token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - journal_id=collection_id, - holder_type="user", - holder_id=user_id, - permission_list=permissions, - ) - - -def find_user_collection( +def find_user_journal( user_id: uuid.UUID, create_if_not_exists: bool = False, ) -> Tuple[Optional[str], Optional[str]]: """ - Find user collection in Brood resources - Can create new collection if not exists and create_if_not_exists = True + Find user journal (collection) in Brood resources + Can create new journal (collection) if not exists and create_if_not_exists = True """ params = { "type": BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, "user_id": str(user_id), } - logger.info(f"Looking for collection for user {user_id}") + logger.info(f"Looking for journal (collection) for user {user_id}") try: user_entity_resources: BugoutResources = bc.list_resources( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, params=params @@ -147,18 +132,16 @@ def find_user_collection( ) if len(user_entity_resources.resources) > 0: - collection_id = user_entity_resources.resources[0].resource_data[ - "collection_id" - ] + journal_id = user_entity_resources.resources[0].resource_data["collection_id"] logger.info( - f"Collection found for user {user_id}. collection_id: {collection_id}" + f"Journal (collection) found for user {user_id}. journal_id: {journal_id}" ) - return collection_id, str(user_entity_resources.resources[0].id) + return journal_id, str(user_entity_resources.resources[0].id) elif create_if_not_exists: - # Create collection new collection for user - logger.info(f"Creating new collection") - collection = create_collection_for_user(user_id) - return collection, None + # Create new journal for user + logger.info(f"Creating new journal (collection)") + journal_id = create_journal_for_user(user_id) + return journal_id, None return None, None @@ -223,33 +206,35 @@ def generate_entity_subscriptions_from_brood_resources() -> None: logger.info(f"parsed users: {len(users_subscriptions)}") - ### Create collections and add subscriptions + ### Create journals (collections) and add subscriptions try: for user_id, subscriptions in users_subscriptions.items(): user_id = str(user_id) - collection_id = None + journal_id = None resource_id_of_user_collection = None - ### Collection can already exist in stages.json + ### Journal can already exist in stages.json if "collection_id" in stages[user_id]: - collection_id = stages[user_id]["collection_id"] + journal_id = stages[user_id]["collection_id"] if "subscription_resource_id" in stages[user_id]: resource_id_of_user_collection = stages[user_id][ "subscription_resource_id" ] else: ### look for collection in brood resources - collection_id, resource_id_of_user_collection = find_user_collection( + journal_id, resource_id_of_user_collection = find_user_journal( user_id, create_if_not_exists=True ) - if collection_id is None: - logger.info(f"Collection not found or create for user {user_id}") + if journal_id is None: + logger.info( + f"Journal (collection) not found or create for user {user_id}" + ) continue - stages[user_id]["collection_id"] = collection_id + stages[user_id]["collection_id"] = journal_id # Create user subscription collection resource @@ -262,7 +247,7 @@ def generate_entity_subscriptions_from_brood_resources() -> None: resource_data = { "type": BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, "user_id": str(user_id), - "collection_id": str(collection_id), + "collection_id": str(journal_id), "version": "1.0.0", } @@ -318,11 +303,11 @@ def generate_entity_subscriptions_from_brood_resources() -> None: # Add subscription to collection - logger.info(f"Add subscription to collection: {collection_id}") + logger.info(f"Add subscription to journal (collection): {journal_id}") entity = add_entity_subscription( user_id=user_id, - collection_id=collection_id, + journal_id=journal_id, subscription_type_id=subscription_type_id, address=address, color=color, @@ -331,7 +316,7 @@ def generate_entity_subscriptions_from_brood_resources() -> None: ) stages[user_id]["processed_subscriptions"][ str(subscription["subscription_id"]) - ] = {"entity_id": str(entity.entity_id), "dashboard_ids": []} + ] = {"entity_id": str(entity.id), "dashboard_ids": []} # Add permissions to user @@ -342,7 +327,7 @@ def generate_entity_subscriptions_from_brood_resources() -> None: try: bc.update_journal_scopes( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - journal_id=collection_id, + journal_id=journal_id, holder_type="user", holder_id=user_id, permission_list=[ @@ -361,12 +346,12 @@ def generate_entity_subscriptions_from_brood_resources() -> None: continue else: logger.warn( - f"User {user_id} == {admin_user_id} permissions not changed. Unexpected behaivior!" + f"User {user_id} == {admin_user_id} permissions not changed. Unexpected behavior!" ) except Exception as e: traceback.print_exc() - logger.error(f"Failed to proccess user subscriptions: {str(e)}") + logger.error(f"Failed to process user subscriptions: {str(e)}") finally: try: with open("stages.json", "w") as f: @@ -561,18 +546,18 @@ def delete_generated_entity_subscriptions_from_brood_resources(): logger.info(f"parsed users: {len(users_subscriptions)}") - ### Create collections and add subscriptions + ### Create journals and add subscriptions try: for user_id, _ in users_subscriptions.items(): user_id = str(user_id) - collection_id = None + journal_id = None resource_id_of_user_collection = None ### Collection can already exist in stages.json if "collection_id" in stages[user_id]: - collection_id = stages[user_id]["collection_id"] + journal_id = stages[user_id]["collection_id"] if "subscription_resource_id" in stages[user_id]: resource_id_of_user_collection = stages[user_id][ @@ -581,35 +566,37 @@ def delete_generated_entity_subscriptions_from_brood_resources(): else: ### look for collection in brood resources - collection_id, resource_id_of_user_collection = find_user_collection( + journal_id, resource_id_of_user_collection = find_user_journal( user_id, create_if_not_exists=False ) - if collection_id is None: + if journal_id is None: logger.info(f"Collection not found or create for user {user_id}") continue ### Delete collection try: - ec.delete_collection( - token=MOONSTREAM_ADMIN_ACCESS_TOKEN, collection_id=collection_id + bc.delete_journal( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=journal_id ) - logger.info(f"Collection deleted {collection_id}") + logger.info(f"Journal (collection) deleted {journal_id}") except Exception as e: - logger.error(f"Failed to delete collection: {str(e)}") + logger.error(f"Failed to delete journal (collection): {str(e)}") ### Delete collection resource try: - logger.info(f"Collection resource id {resource_id_of_user_collection}") + logger.info( + f"Journal (collection) resource id {resource_id_of_user_collection}" + ) bc.delete_resource( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, resource_id=resource_id_of_user_collection, ) logger.info( - f"Collection resource deleted {resource_id_of_user_collection}" + f"Journal (collection) resource deleted {resource_id_of_user_collection}" ) # clear stages @@ -617,12 +604,14 @@ def delete_generated_entity_subscriptions_from_brood_resources(): stages[user_id] = {} except Exception as e: - logger.error(f"Failed to delete collection resource: {str(e)}") + logger.error( + f"Failed to delete journal (collection) resource: {str(e)}" + ) continue except Exception as e: traceback.print_exc() - logger.error(f"Failed to proccess user subscriptions: {str(e)}") + logger.error(f"Failed to process user subscriptions: {str(e)}") def restore_dashboard_state(): @@ -659,7 +648,7 @@ def restore_dashboard_state(): dashboards_by_user[user_id].append(dashboard) - ### Retunr all dashboards to old state + ### Return all dashboards to old state logger.info(f"Amount of users: {len(dashboards_by_user)}") @@ -738,41 +727,41 @@ def fix_duplicates_keys_in_entity_subscription(): timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) - # get collection ids from that resources + # get journal ids from that resources - collection_id_user_id_mappig = {} + collection_id_user_id_mapping = {} for subscription in subscriptions.resources: if "collection_id" in subscription.resource_data: if ( subscription.resource_data["collection_id"] - not in collection_id_user_id_mappig + not in collection_id_user_id_mapping ): - collection_id_user_id_mappig[ + collection_id_user_id_mapping[ subscription.resource_data["collection_id"] ] = subscription.resource_data["user_id"] else: raise Exception( f"Duplicate collection_id {subscription.resource_data['collection_id']} in subscriptions" ) - # go through all collections and fix entities. + # go through all journals and fix entities. # Will creating one new entity with same data but without "type:subscription" in required_fields - for collection_id, user_id in collection_id_user_id_mappig.items(): - # get collection entities - - collection_entities = ec.search_entities( + for journal_id, user_id in collection_id_user_id_mapping.items(): + # get journal entities + journal_entities = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - collection_id=collection_id, + journal_id=journal_id, required_field=[f"type:subscription"], limit=1000, + representation="entity", ) logger.info( - f"Amount of entities in user: {user_id} collection {collection_id}: {len(collection_entities.entities)}" + f"Amount of entities in user: {user_id} journal (collection) {journal_id}: {len(journal_entities.entities)}" ) - for entity in collection_entities.entities: + for entity in journal_entities.entities: # get entity data if entity.secondary_fields is None: @@ -813,43 +802,43 @@ def fix_duplicates_keys_in_entity_subscription(): ) new_required_fields.append({"entity_id": str(entity_id)}) - new_entity = ec.add_entity( + new_entity = bc.create_entity( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - collection_id=collection_id, + journal_id=journal_id, blockchain=entity.blockchain, address=entity.address, - name=entity.name, + title=entity.title, required_fields=new_required_fields, secondary_fields=entity.secondary_fields, ) logger.info( - f"Entity {new_entity.entity_id} created successfully for collection {collection_id}" + f"Entity {new_entity.entity_id} created successfully for journal (collection) {journal_id}" ) except Exception as e: logger.error( - f"Failed to create entity {entity_id} for collection {collection_id}: {str(e)}, user_id: {user_id}" + f"Failed to create entity {entity_id} for journal (collection) {journal_id}: {str(e)}, user_id: {user_id}" ) continue # Update old entity without secondary_fields duplicate try: - ec.update_entity( + bc.update_entity( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - collection_id=collection_id, + journal_id=journal_id, entity_id=entity_id, blockchain=entity.blockchain, address=entity.address, - name=entity.name, + title=entity.title, required_fields=entity.required_fields, secondary_fields=secondary_fields, ) logger.info( - f"Entity {entity_id} updated successfully for collection {collection_id}" + f"Entity {entity_id} updated successfully for journal (collection) {journal_id}" ) except Exception as e: logger.error( - f"Failed to update entity {entity_id} for collection {collection_id}: {str(e)}, user_id: {user_id}" + f"Failed to update entity {entity_id} for journal (collection) {journal_id}: {str(e)}, user_id: {user_id}" ) diff --git a/moonstreamapi/moonstreamapi/admin/queries.py b/moonstreamapi/moonstreamapi/admin/queries.py index cee48772..b119abb0 100644 --- a/moonstreamapi/moonstreamapi/admin/queries.py +++ b/moonstreamapi/moonstreamapi/admin/queries.py @@ -1,25 +1,23 @@ import argparse -from collections import Counter import json +import logging +import textwrap +from typing import Any, Dict from bugout.data import BugoutResources from bugout.exceptions import BugoutResponseException from moonstream.client import Moonstream # type: ignore -import logging -from typing import Dict, Any -import textwrap from sqlalchemy import text - +from ..actions import get_all_entries_from_search, name_normalization from ..data import BUGOUT_RESOURCE_QUERY_RESOLVER from ..settings import ( BUGOUT_REQUEST_TIMEOUT_SECONDS, MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_QUERIES_JOURNAL_ID, + MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE, ) -from ..settings import bugout_client as bc, MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE -from ..actions import get_all_entries_from_search, name_normalization - +from ..settings import bugout_client as bc logger = logging.getLogger(__name__) diff --git a/moonstreamapi/moonstreamapi/data.py b/moonstreamapi/moonstreamapi/data.py index 699024df..f58a4cc1 100644 --- a/moonstreamapi/moonstreamapi/data.py +++ b/moonstreamapi/moonstreamapi/data.py @@ -1,10 +1,10 @@ """ Pydantic schemas for the Moonstream HTTP API """ -from datetime import datetime import json +from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Optional, Union, Literal +from typing import Any, Dict, List, Literal, Optional, Union from uuid import UUID from xmlrpc.client import Boolean @@ -12,7 +12,6 @@ from fastapi import Form from pydantic import BaseModel, Field, validator from sqlalchemy import false - USER_ONBOARDING_STATE = "onboarding_state" BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver" @@ -244,7 +243,7 @@ class OnboardingState(BaseModel): steps: Dict[str, int] -class SubdcriptionsAbiResponse(BaseModel): +class SubscriptionsAbiResponse(BaseModel): abi: str diff --git a/moonstreamapi/moonstreamapi/routes/dashboards.py b/moonstreamapi/moonstreamapi/routes/dashboards.py index 4b047f05..b8de9c99 100644 --- a/moonstreamapi/moonstreamapi/routes/dashboards.py +++ b/moonstreamapi/moonstreamapi/routes/dashboards.py @@ -1,6 +1,5 @@ import json import logging -from os import read from typing import Any, Dict, List, Optional, Union from uuid import UUID @@ -16,15 +15,16 @@ from ..middleware import MoonstreamHTTPException from ..reporter import reporter from ..settings import ( BUGOUT_REQUEST_TIMEOUT_SECONDS, + BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_APPLICATION_ID, - MOONSTREAM_CRAWLERS_SERVER_URL, MOONSTREAM_CRAWLERS_SERVER_PORT, + MOONSTREAM_CRAWLERS_SERVER_URL, MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, - BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, ) -from ..settings import bugout_client as bc, entity_client as ec +from ..settings import bugout_client as bc +from ..settings import entity_client as ec logger = logging.getLogger(__name__) @@ -52,19 +52,20 @@ async def add_dashboard_handler( subscription_settings = dashboard.subscription_settings - # Get user collection id + # Get user journal (collection) id - collection_id = actions.get_entity_subscription_collection_id( + journal_id = actions.get_entity_subscription_journal_id( resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, user_id=user.id, token=MOONSTREAM_ADMIN_ACCESS_TOKEN, ) - subscriprions_list = ec.search_entities( + subscriprions_list = bc.search( token=token, - collection_id=collection_id, + journal_id=journal_id, required_field=[f"type:subscription"], limit=1000, + representation="entity", ) # process existing subscriptions with supplied ids @@ -137,7 +138,7 @@ async def add_dashboard_handler( tags=["subscriptions"], response_model=BugoutResource, ) -async def delete_subscription_handler(request: Request, dashboard_id: str): +async def delete_subscription_handler(request: Request, dashboard_id: str = Path(...)): """ Delete subscriptions. """ @@ -181,9 +182,9 @@ async def get_dashboards_handler( return resources -@router.get("/{dashboarsd_id}", tags=["dashboards"], response_model=BugoutResource) +@router.get("/{dashboard_id}", tags=["dashboards"], response_model=BugoutResource) async def get_dashboard_handler( - request: Request, dashboarsd_id: UUID + request: Request, dashboard_id: UUID = Path(...) ) -> BugoutResource: """ Get user's subscriptions. @@ -193,7 +194,7 @@ async def get_dashboard_handler( try: resource: BugoutResource = bc.get_resource( token=token, - resource_id=dashboarsd_id, + resource_id=dashboard_id, timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) except BugoutResponseException as e: @@ -211,7 +212,7 @@ async def get_dashboard_handler( @router.put("/{dashboard_id}", tags=["dashboards"], response_model=BugoutResource) async def update_dashboard_handler( request: Request, - dashboard_id: str, + dashboard_id: str = Path(...), dashboard: data.DashboardUpdate = Body(...), ) -> BugoutResource: """ @@ -224,19 +225,20 @@ async def update_dashboard_handler( subscription_settings = dashboard.subscription_settings - # Get user collection id + # Get user journal (collection) id - collection_id = actions.get_entity_subscription_collection_id( + journal_id = actions.get_entity_subscription_journal_id( resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, user_id=user.id, token=MOONSTREAM_ADMIN_ACCESS_TOKEN, ) - subscriprions_list = ec.search_entities( + subscriprions_list = bc.search( token=token, - collection_id=collection_id, + journal_id=journal_id, required_field=[f"type:subscription"], limit=1000, + representation="entity", ) available_subscriptions_ids: Dict[Union[UUID, str], EntityResponse] = { @@ -301,7 +303,7 @@ async def update_dashboard_handler( @router.get("/{dashboard_id}/stats", tags=["dashboards"]) async def get_dashboard_data_links_handler( - request: Request, dashboard_id: str + request: Request, dashboard_id: str = Path(...) ) -> Dict[Union[UUID, str], Any]: """ Get s3 presign urls for dashboard grafics @@ -328,17 +330,18 @@ async def get_dashboard_data_links_handler( # get subscriptions - collection_id = actions.get_entity_subscription_collection_id( + journal_id = actions.get_entity_subscription_journal_id( resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, user_id=user.id, token=MOONSTREAM_ADMIN_ACCESS_TOKEN, ) - subscriprions_list = ec.search_entities( + subscriprions_list = bc.search( token=token, - collection_id=collection_id, + journal_id=journal_id, required_field=[f"type:subscription"], limit=1000, + representation="entity", ) # filter out dasboards diff --git a/moonstreamapi/moonstreamapi/routes/subscriptions.py b/moonstreamapi/moonstreamapi/routes/subscriptions.py index 257c387e..a15ddd2e 100644 --- a/moonstreamapi/moonstreamapi/routes/subscriptions.py +++ b/moonstreamapi/moonstreamapi/routes/subscriptions.py @@ -1,42 +1,38 @@ """ The Moonstream subscriptions HTTP API """ -from concurrent.futures import as_completed, ProcessPoolExecutor, ThreadPoolExecutor import hashlib import json import logging +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed from typing import Any, Dict, List, Optional -from bugout.exceptions import BugoutResponseException from bugout.data import BugoutSearchResult -from fastapi import APIRouter, Depends, Request, Form, BackgroundTasks +from bugout.exceptions import BugoutResponseException +from fastapi import APIRouter, BackgroundTasks, Depends, Form, Path, Query, Request from moonstreamdb.blockchain import AvailableBlockchainType from web3 import Web3 +from .. import data from ..actions import ( AddressNotSmartContractException, - validate_abi_json, + EntityJournalNotFoundException, apply_moonworm_tasks, - get_entity_subscription_collection_id, - EntityCollectionNotFoundException, check_if_smart_contract, + get_entity_subscription_journal_id, get_list_of_support_interfaces, + validate_abi_json, ) from ..admin import subscription_types -from .. import data -from ..admin import subscription_types from ..middleware import MoonstreamHTTPException from ..reporter import reporter -from ..settings import bugout_client as bc, entity_client as ec from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_ENTITIES_RESERVED_TAGS, THREAD_TIMEOUT_SECONDS, ) -from ..web3_provider import ( - yield_web3_provider, -) - +from ..settings import bugout_client as bc +from ..web3_provider import yield_web3_provider logger = logging.getLogger(__name__) @@ -160,36 +156,35 @@ async def add_subscription_handler( required_fields.extend(allowed_required_fields) try: - collection_id = get_entity_subscription_collection_id( + journal_id = get_entity_subscription_journal_id( resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, token=MOONSTREAM_ADMIN_ACCESS_TOKEN, user_id=user.id, create_if_not_exist=True, ) - - entity = ec.add_entity( + entity = bc.create_entity( token=token, - collection_id=collection_id, + journal_id=journal_id, address=address, blockchain=subscription_types.CANONICAL_SUBSCRIPTION_TYPES[ subscription_type_id ].blockchain, - name=label, + title=label, required_fields=required_fields, secondary_fields=content, ) - except EntityCollectionNotFoundException as e: + except EntityJournalNotFoundException as e: raise MoonstreamHTTPException( status_code=404, - detail="User subscriptions collection not found", + detail="User subscriptions journal not found", internal_error=e, ) except Exception as e: - logger.error(f"Failed to get collection id") + logger.error(f"Failed to get journal id") raise MoonstreamHTTPException( status_code=500, internal_error=e, - detail="Currently unable to get collection id", + detail="Currently unable to get journal id", ) normalized_entity_tags = [ @@ -200,7 +195,7 @@ async def add_subscription_handler( ] return data.SubscriptionResourceData( - id=str(entity.entity_id), + id=str(entity.id), user_id=str(user.id), address=address, color=color, @@ -219,28 +214,29 @@ async def add_subscription_handler( tags=["subscriptions"], response_model=data.SubscriptionResourceData, ) -async def delete_subscription_handler(request: Request, subscription_id: str): +async def delete_subscription_handler( + request: Request, subscription_id: str = Path(...) +): """ Delete subscriptions. """ token = request.state.token user = request.state.user try: - collection_id = get_entity_subscription_collection_id( + journal_id = get_entity_subscription_journal_id( resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, token=MOONSTREAM_ADMIN_ACCESS_TOKEN, user_id=user.id, ) - - deleted_entity = ec.delete_entity( + deleted_entity = bc.delete_entity( token=token, - collection_id=collection_id, + journal_id=journal_id, entity_id=subscription_id, ) - except EntityCollectionNotFoundException as e: + except EntityJournalNotFoundException as e: raise MoonstreamHTTPException( status_code=404, - detail="User subscriptions collection not found", + detail="User subscriptions journal not found", internal_error=e, ) except Exception as e: @@ -272,7 +268,7 @@ async def delete_subscription_handler(request: Request, subscription_id: str): abi = deleted_entity.secondary_fields.get("abi") return data.SubscriptionResourceData( - id=str(deleted_entity.entity_id), + id=str(deleted_entity.id), user_id=str(user.id), address=deleted_entity.address, color=color, @@ -289,8 +285,8 @@ async def delete_subscription_handler(request: Request, subscription_id: str): @router.get("/", tags=["subscriptions"], response_model=data.SubscriptionsListResponse) async def get_subscriptions_handler( request: Request, - limit: Optional[int] = 10, - offset: Optional[int] = 0, + limit: Optional[int] = Query(10), + offset: Optional[int] = Query(0), ) -> data.SubscriptionsListResponse: """ Get user's subscriptions. @@ -298,25 +294,24 @@ async def get_subscriptions_handler( token = request.state.token user = request.state.user try: - collection_id = get_entity_subscription_collection_id( + journal_id = get_entity_subscription_journal_id( resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, token=MOONSTREAM_ADMIN_ACCESS_TOKEN, user_id=user.id, create_if_not_exist=True, ) - - subscriprions_list = ec.search_entities( + subscriptions_list = bc.search( token=token, - collection_id=collection_id, - required_field=[f"type:subscription"], + journal_id=journal_id, + query="tag:type:subscription", limit=limit, offset=offset, ) - except EntityCollectionNotFoundException as e: + except EntityJournalNotFoundException as e: raise MoonstreamHTTPException( status_code=404, - detail="User subscriptions collection not found", + detail="User subscriptions journal not found", internal_error=e, ) except Exception as e: @@ -328,7 +323,7 @@ async def get_subscriptions_handler( subscriptions = [] - for subscription in subscriprions_list.entities: + for subscription in subscriptions_list.entities: tags = subscription.required_fields label, color, subscription_type_id = None, None, None @@ -352,7 +347,7 @@ async def get_subscriptions_handler( subscriptions.append( data.SubscriptionResourceData( - id=str(subscription.entity_id), + id=str(subscription.id), user_id=str(user.id), address=subscription.address, color=color, @@ -378,8 +373,8 @@ async def get_subscriptions_handler( ) async def update_subscriptions_handler( request: Request, - subscription_id: str, background_tasks: BackgroundTasks, + subscription_id: str = Path(...), ) -> data.SubscriptionResourceData: """ Get user's subscriptions. @@ -401,16 +396,16 @@ async def update_subscriptions_handler( tags = form_data.tags try: - collection_id = get_entity_subscription_collection_id( + journal_id = get_entity_subscription_journal_id( resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, token=MOONSTREAM_ADMIN_ACCESS_TOKEN, user_id=user.id, ) # get subscription entity - subscription_entity = ec.get_entity( + subscription_entity = bc.get_entity( token=token, - collection_id=collection_id, + journal_id=journal_id, entity_id=subscription_id, ) @@ -430,17 +425,17 @@ async def update_subscriptions_handler( if not subscription_type_id: logger.error( - f"Subscription entity {subscription_id} in collection {collection_id} has no subscription_type_id malformed subscription entity" + f"Subscription entity {subscription_id} in journal (collection) {journal_id} has no subscription_type_id malformed subscription entity" ) raise MoonstreamHTTPException( status_code=409, detail="Not valid subscription entity", ) - except EntityCollectionNotFoundException as e: + except EntityJournalNotFoundException as e: raise MoonstreamHTTPException( status_code=404, - detail="User subscriptions collection not found", + detail="User subscriptions journal not found", internal_error=e, ) except Exception as e: @@ -488,17 +483,16 @@ async def update_subscriptions_handler( if allowed_required_fields: update_required_fields.extend(allowed_required_fields) try: - subscription = ec.update_entity( + subscription = bc.update_entity( token=token, - collection_id=collection_id, + journal_id=journal_id, entity_id=subscription_id, + title=subscription_entity.title, address=subscription_entity.address, blockchain=subscription_entity.blockchain, - name=subscription_entity.name, required_fields=update_required_fields, secondary_fields=update_secondary_fields, ) - except Exception as e: logger.error(f"Error update user subscriptions: {str(e)}") raise MoonstreamHTTPException(status_code=500, internal_error=e) @@ -519,7 +513,7 @@ async def update_subscriptions_handler( ] return data.SubscriptionResourceData( - id=str(subscription.entity_id), + id=str(subscription.id), user_id=str(user.id), address=subscription.address, color=color, @@ -536,33 +530,33 @@ async def update_subscriptions_handler( @router.get( "/{subscription_id}/abi", tags=["subscriptions"], - response_model=data.SubdcriptionsAbiResponse, + response_model=data.SubscriptionsAbiResponse, ) async def get_subscription_abi_handler( request: Request, - subscription_id: str, -) -> data.SubdcriptionsAbiResponse: + subscription_id: str = Path(...), +) -> data.SubscriptionsAbiResponse: token = request.state.token user = request.state.user try: - collection_id = get_entity_subscription_collection_id( + journal_id = get_entity_subscription_journal_id( resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, token=MOONSTREAM_ADMIN_ACCESS_TOKEN, user_id=user.id, ) # get subscription entity - subscription_resource = ec.get_entity( + subscription_resource = bc.get_entity( token=token, - collection_id=collection_id, + journal_id=journal_id, entity_id=subscription_id, ) - except EntityCollectionNotFoundException as e: + except EntityJournalNotFoundException as e: raise MoonstreamHTTPException( status_code=404, - detail="User subscriptions collection not found", + detail="User subscriptions journal not found", internal_error=e, ) except Exception as e: @@ -572,7 +566,7 @@ async def get_subscription_abi_handler( if "abi" not in subscription_resource.secondary_fields.keys(): raise MoonstreamHTTPException(status_code=404, detail="Abi not found") - return data.SubdcriptionsAbiResponse( + return data.SubscriptionsAbiResponse( abi=subscription_resource.secondary_fields["abi"] ) @@ -605,7 +599,7 @@ async def list_subscription_types() -> data.SubscriptionTypesListResponse: tags=["subscriptions"], response_model=data.ContractInfoResponse, ) -async def address_info(request: Request, address: str): +async def address_info(request: Request, address: str = Query(...)): """ Looking if address is contract """ @@ -668,8 +662,8 @@ async def address_info(request: Request, address: str): ) def get_contract_interfaces( request: Request, - address: str, - blockchain: str, + address: str = Query(...), + blockchain: str = Query(...), ): """ Request contract interfaces from web3 diff --git a/moonstreamapi/moonstreamapi/routes/users.py b/moonstreamapi/moonstreamapi/routes/users.py index 6b97cccd..b54149de 100644 --- a/moonstreamapi/moonstreamapi/routes/users.py +++ b/moonstreamapi/moonstreamapi/routes/users.py @@ -10,7 +10,7 @@ from bugout.exceptions import BugoutResponseException from fastapi import APIRouter, Body, Form, Request from .. import data -from ..actions import create_onboarding_resource, generate_collection_for_user +from ..actions import create_onboarding_resource from ..middleware import MoonstreamHTTPException from ..settings import BUGOUT_REQUEST_TIMEOUT_SECONDS, MOONSTREAM_APPLICATION_ID from ..settings import bugout_client as bc diff --git a/moonstreamapi/requirements.txt b/moonstreamapi/requirements.txt index fe11db45..000cff17 100644 --- a/moonstreamapi/requirements.txt +++ b/moonstreamapi/requirements.txt @@ -9,7 +9,7 @@ base58==2.1.1 bitarray==2.6.0 boto3==1.26.5 botocore==1.29.5 -bugout>=0.2.10 +bugout>=0.2.12 certifi==2022.9.24 charset-normalizer==2.1.1 click==8.1.3 diff --git a/moonstreamapi/setup.py b/moonstreamapi/setup.py index c23a227a..00be79ed 100644 --- a/moonstreamapi/setup.py +++ b/moonstreamapi/setup.py @@ -13,12 +13,12 @@ setup( install_requires=[ "appdirs", "boto3", - "bugout>=0.2.10", + "bugout>=0.2.12", "moonstream-entity>=0.0.5", "fastapi", "moonstreamdb>=0.3.4", "humbug", - "pydantic", + "pydantic==1.10.2", "pyevmasm", "python-dateutil", "python-multipart", From bbe1530eef7fe84c14e87a9afd2566f64b2e99dc Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 31 Jul 2023 12:36:48 +0000 Subject: [PATCH 04/50] Removed entity client dependency --- .../migrations/generate_entity_subscriptions.py | 17 ++++++----------- .../moonstreamapi/routes/dashboards.py | 1 - moonstreamapi/moonstreamapi/settings.py | 14 +------------- moonstreamapi/requirements.txt | 1 - moonstreamapi/setup.py | 1 - 5 files changed, 7 insertions(+), 27 deletions(-) diff --git a/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py b/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py index c63459d8..11ab5961 100644 --- a/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py +++ b/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py @@ -774,12 +774,7 @@ def fix_duplicates_keys_in_entity_subscription(): secondary_fields = secondary_fields["secondary_fields"] - # get entity id - - entity_id = entity.entity_id - # get entity type - entity_type = None # extract required fields @@ -800,7 +795,7 @@ def fix_duplicates_keys_in_entity_subscription(): new_required_fields.append( {"type": "copy_of_malformed_entity_20230213"} ) - new_required_fields.append({"entity_id": str(entity_id)}) + new_required_fields.append({"entity_id": str(entity.id)}) new_entity = bc.create_entity( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -812,12 +807,12 @@ def fix_duplicates_keys_in_entity_subscription(): secondary_fields=entity.secondary_fields, ) logger.info( - f"Entity {new_entity.entity_id} created successfully for journal (collection) {journal_id}" + f"Entity {new_entity.id} created successfully for journal (collection) {journal_id}" ) except Exception as e: logger.error( - f"Failed to create entity {entity_id} for journal (collection) {journal_id}: {str(e)}, user_id: {user_id}" + f"Failed to create entity {entity.id} for journal (collection) {journal_id}: {str(e)}, user_id: {user_id}" ) continue @@ -827,7 +822,7 @@ def fix_duplicates_keys_in_entity_subscription(): bc.update_entity( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=journal_id, - entity_id=entity_id, + entity_id=entity.id, blockchain=entity.blockchain, address=entity.address, title=entity.title, @@ -835,10 +830,10 @@ def fix_duplicates_keys_in_entity_subscription(): secondary_fields=secondary_fields, ) logger.info( - f"Entity {entity_id} updated successfully for journal (collection) {journal_id}" + f"Entity {entity.id} updated successfully for journal (collection) {journal_id}" ) except Exception as e: logger.error( - f"Failed to update entity {entity_id} for journal (collection) {journal_id}: {str(e)}, user_id: {user_id}" + f"Failed to update entity {entity.id} for journal (collection) {journal_id}: {str(e)}, user_id: {user_id}" ) diff --git a/moonstreamapi/moonstreamapi/routes/dashboards.py b/moonstreamapi/moonstreamapi/routes/dashboards.py index b8de9c99..2bb13278 100644 --- a/moonstreamapi/moonstreamapi/routes/dashboards.py +++ b/moonstreamapi/moonstreamapi/routes/dashboards.py @@ -24,7 +24,6 @@ from ..settings import ( MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, ) from ..settings import bugout_client as bc -from ..settings import entity_client as ec logger = logging.getLogger(__name__) diff --git a/moonstreamapi/moonstreamapi/settings.py b/moonstreamapi/moonstreamapi/settings.py index 1d11b1c8..b79cdadd 100644 --- a/moonstreamapi/moonstreamapi/settings.py +++ b/moonstreamapi/moonstreamapi/settings.py @@ -1,12 +1,9 @@ import os -from typing import Optional, Dict -from uuid import UUID +from typing import Dict, Optional from bugout.app import Bugout -from entity.client import Entity # type: ignore from moonstreamdb.blockchain import AvailableBlockchainType - # Bugout BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev") BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev") @@ -14,15 +11,6 @@ BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev" bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL) -# Entity - -MOONSTREAM_ENTITY_URL = os.environ.get("MOONSTREAM_ENTITY_URL", "") -if MOONSTREAM_ENTITY_URL == "": - raise ValueError("MOONSTREAM_ENTITY_URL environment variable must be set") - -entity_client = Entity(MOONSTREAM_ENTITY_URL) - - BUGOUT_REQUEST_TIMEOUT_SECONDS = 5 HUMBUG_REPORTER_BACKEND_TOKEN = os.environ.get("HUMBUG_REPORTER_BACKEND_TOKEN") diff --git a/moonstreamapi/requirements.txt b/moonstreamapi/requirements.txt index 000cff17..73c5b18d 100644 --- a/moonstreamapi/requirements.txt +++ b/moonstreamapi/requirements.txt @@ -36,7 +36,6 @@ jsonschema==4.17.0 lru-dict==1.1.8 Mako==1.2.3 MarkupSafe==2.1.1 -moonstream-entity==0.0.5 moonstreamdb==0.3.4 multiaddr==0.0.9 multidict==6.0.2 diff --git a/moonstreamapi/setup.py b/moonstreamapi/setup.py index 00be79ed..f58ea49f 100644 --- a/moonstreamapi/setup.py +++ b/moonstreamapi/setup.py @@ -14,7 +14,6 @@ setup( "appdirs", "boto3", "bugout>=0.2.12", - "moonstream-entity>=0.0.5", "fastapi", "moonstreamdb>=0.3.4", "humbug", From 5dfcfd634ffbde8e93ed4616afee8a522d73b734 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 31 Jul 2023 14:19:22 +0000 Subject: [PATCH 05/50] Fixes to work with spire entity --- moonstreamapi/moonstreamapi/actions.py | 24 ++++++------ .../generate_entity_subscriptions.py | 5 ++- moonstreamapi/moonstreamapi/data.py | 7 ---- .../moonstreamapi/providers/bugout.py | 12 +++--- .../moonstreamapi/routes/dashboards.py | 21 +++++----- .../moonstreamapi/routes/subscriptions.py | 38 +++++++++++++------ .../moonstreamapi/selectors_storage.py | 4 +- moonstreamapi/mypy.ini | 6 +++ 8 files changed, 66 insertions(+), 51 deletions(-) diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index 7c908126..16b6279c 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -477,7 +477,7 @@ def get_all_entries_from_search( results: List[BugoutSearchResult] = [] - existing_metods = bc.search( + existing_methods = bc.search( token=token, journal_id=journal_id, query=search_query, @@ -486,11 +486,11 @@ def get_all_entries_from_search( limit=limit, offset=offset, ) - results.extend(existing_metods.results) + results.extend(existing_methods.results) # type: ignore - if len(results) != existing_metods.total_results: - for offset in range(limit, existing_metods.total_results, limit): - existing_metods = bc.search( + if len(results) != existing_methods.total_results: + for offset in range(limit, existing_methods.total_results, limit): + existing_methods = bc.search( token=token, journal_id=journal_id, query=search_query, @@ -499,7 +499,7 @@ def get_all_entries_from_search( limit=limit, offset=offset, ) - results.extend(existing_metods.results) + results.extend(existing_methods.results) # type: ignore return results @@ -641,7 +641,7 @@ def get_entity_subscription_journal_id( token: Union[uuid.UUID, str], user_id: uuid.UUID, create_if_not_exist: bool = False, -) -> Optional[str]: +) -> str: """ Get collection_id (journal_id) from brood resources. If journal not exist and create_if_not_exist is True """ @@ -684,7 +684,7 @@ def generate_journal_for_user( journals: BugoutJournals = bc.list_journals(token=token) available_journals: Dict[str, str] = { - journal.name: journal.id for journal in journals.journals + journal.name: str(journal.id) for journal in journals.journals } subscription_journal_name = f"subscriptions_{user_id}" @@ -693,7 +693,7 @@ def generate_journal_for_user( journal: BugoutJournal = bc.create_journal( token=token, name=subscription_journal_name ) - journal_id = journal.id + journal_id = str(journal.id) else: journal_id = available_journals[subscription_journal_name] except Exception as e: @@ -705,7 +705,7 @@ def generate_journal_for_user( resource_data = { "type": resource_type, "user_id": str(user_id), - "collection_id": str(journal_id), + "collection_id": journal_id, } try: @@ -832,14 +832,14 @@ def get_list_of_support_interfaces( list_of_interfaces.sort() - for interaface in list_of_interfaces: + for interface in list_of_interfaces: calls.append( ( contract.address, FunctionSignature( contract.get_function_by_name("supportsInterface") ) - .encode_data([bytes.fromhex(interaface)]) + .encode_data([bytes.fromhex(interface)]) .hex(), ) ) diff --git a/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py b/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py index 11ab5961..a192197e 100644 --- a/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py +++ b/moonstreamapi/moonstreamapi/admin/migrations/generate_entity_subscriptions.py @@ -68,7 +68,8 @@ def add_entity_subscription( f"Unknown subscription type ID: {subscription_type_id}. " f"Known subscription type IDs: {CANONICAL_SUBSCRIPTION_TYPES.keys()}" ) - elif CANONICAL_SUBSCRIPTION_TYPES[subscription_type_id].blockchain is None: + blockchain = CANONICAL_SUBSCRIPTION_TYPES[subscription_type_id].blockchain + if blockchain is None: raise ValueError( f"Subscription type ID {subscription_type_id} is not a blockchain subscription type." ) @@ -77,7 +78,7 @@ def add_entity_subscription( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=journal_id, address=address, - blockchain=CANONICAL_SUBSCRIPTION_TYPES[subscription_type_id].blockchain, + blockchain=blockchain, title=label, required_fields=[ {"type": "subscription"}, diff --git a/moonstreamapi/moonstreamapi/data.py b/moonstreamapi/moonstreamapi/data.py index f58a4cc1..b092e151 100644 --- a/moonstreamapi/moonstreamapi/data.py +++ b/moonstreamapi/moonstreamapi/data.py @@ -57,13 +57,6 @@ class SubscriptionResourceData(BaseModel): updated_at: Optional[datetime] -class CreateSubscriptionRequest(BaseModel): - address: str - color: str - label: str - subscription_type_id: str - - class PingResponse(BaseModel): """ Schema for ping response diff --git a/moonstreamapi/moonstreamapi/providers/bugout.py b/moonstreamapi/moonstreamapi/providers/bugout.py index 18d5c205..f07cb284 100644 --- a/moonstreamapi/moonstreamapi/providers/bugout.py +++ b/moonstreamapi/moonstreamapi/providers/bugout.py @@ -4,10 +4,10 @@ Event providers powered by Bugout journals. import json import logging from datetime import datetime -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Union from bugout.app import Bugout -from bugout.data import BugoutResource, BugoutSearchResult +from bugout.data import BugoutResource, BugoutSearchResult, BugoutSearchResultAsEntity from bugout.journal import SearchOrder from dateutil.parser import isoparse from dateutil.tz import UTC @@ -155,7 +155,7 @@ class BugoutEventProvider: timeout=self.timeout, order=SearchOrder.DESCENDING, ) - events.extend([self.entry_event(entry) for entry in search_results.results]) + events.extend([self.entry_event(entry) for entry in search_results.results]) # type: ignore offset = search_results.next_offset return stream_boundary, events @@ -192,7 +192,7 @@ class BugoutEventProvider: timeout=self.timeout, order=SearchOrder.DESCENDING, ) - return [self.entry_event(entry) for entry in search_results.results] + return [self.entry_event(entry) for entry in search_results.results] # type: ignore def next_event( self, @@ -233,7 +233,7 @@ class BugoutEventProvider: ) if not search_results.results: return None - return self.entry_event(search_results.results[0]) + return self.entry_event(search_results.results[0]) # type: ignore def previous_event( self, @@ -274,7 +274,7 @@ class BugoutEventProvider: ) if not search_results.results: return None - return self.entry_event(search_results.results[0]) + return self.entry_event(search_results.results[0]) # type: ignore class EthereumTXPoolProvider(BugoutEventProvider): diff --git a/moonstreamapi/moonstreamapi/routes/dashboards.py b/moonstreamapi/moonstreamapi/routes/dashboards.py index 2bb13278..ff66d339 100644 --- a/moonstreamapi/moonstreamapi/routes/dashboards.py +++ b/moonstreamapi/moonstreamapi/routes/dashboards.py @@ -5,9 +5,8 @@ from uuid import UUID import boto3 # type: ignore import requests # type: ignore -from bugout.data import BugoutResource, BugoutResources +from bugout.data import BugoutResource, BugoutResources, BugoutSearchResultAsEntity from bugout.exceptions import BugoutResponseException -from entity.data import EntitiesResponse, EntityResponse # type: ignore from fastapi import APIRouter, Body, Path, Query, Request from .. import actions, data @@ -59,7 +58,7 @@ async def add_dashboard_handler( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, ) - subscriprions_list = bc.search( + subscriptions_list = bc.search( token=token, journal_id=journal_id, required_field=[f"type:subscription"], @@ -69,9 +68,9 @@ async def add_dashboard_handler( # process existing subscriptions with supplied ids - available_subscriptions_ids: Dict[Union[UUID, str], EntityResponse] = { + available_subscriptions_ids: Dict[Union[UUID, str], BugoutSearchResultAsEntity] = { subscription.entity_id: subscription - for subscription in subscriprions_list.entities + for subscription in subscriptions_list.entities } for dashboard_subscription in subscription_settings: @@ -232,7 +231,7 @@ async def update_dashboard_handler( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, ) - subscriprions_list = bc.search( + subscriptions_list = bc.search( token=token, journal_id=journal_id, required_field=[f"type:subscription"], @@ -240,9 +239,9 @@ async def update_dashboard_handler( representation="entity", ) - available_subscriptions_ids: Dict[Union[UUID, str], EntityResponse] = { + available_subscriptions_ids: Dict[Union[UUID, str], BugoutSearchResultAsEntity] = { subscription.entity_id: subscription - for subscription in subscriprions_list.entities + for subscription in subscriptions_list.entities } for dashboard_subscription in subscription_settings: @@ -335,7 +334,7 @@ async def get_dashboard_data_links_handler( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, ) - subscriprions_list = bc.search( + subscriptions_list = bc.search( token=token, journal_id=journal_id, required_field=[f"type:subscription"], @@ -352,9 +351,9 @@ async def get_dashboard_data_links_handler( ] ] - dashboard_subscriptions: Dict[Union[UUID, str], EntitiesResponse] = { + dashboard_subscriptions: Dict[Union[UUID, str], BugoutSearchResultAsEntity] = { subscription.entity_id: subscription - for subscription in subscriprions_list.entities + for subscription in subscriptions_list.entities if str(subscription.entity_id) in subscriptions_ids } diff --git a/moonstreamapi/moonstreamapi/routes/subscriptions.py b/moonstreamapi/moonstreamapi/routes/subscriptions.py index a15ddd2e..734c10c3 100644 --- a/moonstreamapi/moonstreamapi/routes/subscriptions.py +++ b/moonstreamapi/moonstreamapi/routes/subscriptions.py @@ -5,7 +5,7 @@ import hashlib import json import logging from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from bugout.data import BugoutSearchResult from bugout.exceptions import BugoutResponseException @@ -136,7 +136,7 @@ async def add_subscription_handler( if description: content["description"] = description - allowed_required_fields = [] + allowed_required_fields: List[Any] = [] if tags: allowed_required_fields = [ item @@ -144,7 +144,7 @@ async def add_subscription_handler( if not any(key in item for key in MOONSTREAM_ENTITIES_RESERVED_TAGS) ] - required_fields = [ + required_fields: List[Dict[str, Union[str, bool, int, List[Any]]]] = [ {"type": "subscription"}, {"subscription_type_id": f"{subscription_type_id}"}, {"color": f"{color}"}, @@ -162,13 +162,14 @@ async def add_subscription_handler( user_id=user.id, create_if_not_exist=True, ) + blockchain = subscription_types.CANONICAL_SUBSCRIPTION_TYPES[ + subscription_type_id + ].blockchain entity = bc.create_entity( token=token, journal_id=journal_id, address=address, - blockchain=subscription_types.CANONICAL_SUBSCRIPTION_TYPES[ - subscription_type_id - ].blockchain, + blockchain=blockchain if blockchain is not None else "", title=label, required_fields=required_fields, secondary_fields=content, @@ -186,10 +187,15 @@ async def add_subscription_handler( internal_error=e, detail="Currently unable to get journal id", ) - + entity_required_fields = ( + entity.required_fields if entity.required_fields is not None else [] + ) + entity_secondary_fields = ( + entity.secondary_fields if entity.secondary_fields is not None else {} + ) normalized_entity_tags = [ f"{key}:{value}" - for tag in entity.required_fields + for tag in entity_required_fields for key, value in tag.items() if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS ] @@ -200,8 +206,8 @@ async def add_subscription_handler( address=address, color=color, label=label, - abi=entity.secondary_fields.get("abi"), - description=entity.secondary_fields.get("description"), + abi=entity_secondary_fields.get("abi"), + description=entity_secondary_fields.get("description"), tags=normalized_entity_tags, subscription_type_id=subscription_type_id, updated_at=entity.updated_at, @@ -252,6 +258,7 @@ async def delete_subscription_handler( color = None label = None abi = None + description = None if tags is not None: for tag in tags: @@ -266,6 +273,13 @@ async def delete_subscription_handler( if deleted_entity.secondary_fields is not None: abi = deleted_entity.secondary_fields.get("abi") + description = deleted_entity.secondary_fields.get("description") + + deleted_entity_required_fields = ( + deleted_entity.required_fields + if deleted_entity.required_fields is not None + else [] + ) return data.SubscriptionResourceData( id=str(deleted_entity.id), @@ -274,8 +288,8 @@ async def delete_subscription_handler( color=color, label=label, abi=abi, - description=deleted_entity.secondary_fields.get("description"), - tags=deleted_entity.required_fields, + description=description, + tags=deleted_entity_required_fields, subscription_type_id=subscription_type_id, updated_at=deleted_entity.updated_at, created_at=deleted_entity.created_at, diff --git a/moonstreamapi/moonstreamapi/selectors_storage.py b/moonstreamapi/moonstreamapi/selectors_storage.py index a4da64e3..9c5a0c82 100644 --- a/moonstreamapi/moonstreamapi/selectors_storage.py +++ b/moonstreamapi/moonstreamapi/selectors_storage.py @@ -1,4 +1,6 @@ -selectors = { +from typing import Any, Dict + +selectors: Dict[str, Any] = { "274c7b3c": { "name": "ERC20PresetMinterPauser", "selector": "274c7b3c", diff --git a/moonstreamapi/mypy.ini b/moonstreamapi/mypy.ini index 47838c47..7e4bc635 100644 --- a/moonstreamapi/mypy.ini +++ b/moonstreamapi/mypy.ini @@ -8,3 +8,9 @@ ignore_missing_imports = True [mypy-pyevmasm.*] ignore_missing_imports = True + +[mypy-requests.*] +ignore_missing_imports = True + +[mypy-dateutil.*] +ignore_missing_imports = True From 0878d5b7981b753681ef62e260599b0dc5e43eb1 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 31 Jul 2023 15:37:59 +0000 Subject: [PATCH 06/50] Fixed search entities results for spire compatibility --- .../moonstreamapi/routes/dashboards.py | 42 ++++++----- moonstreamapi/moonstreamapi/routes/queries.py | 28 ++++---- .../moonstreamapi/routes/subscriptions.py | 69 ++++++++++++------- 3 files changed, 80 insertions(+), 59 deletions(-) diff --git a/moonstreamapi/moonstreamapi/routes/dashboards.py b/moonstreamapi/moonstreamapi/routes/dashboards.py index ff66d339..594e5d17 100644 --- a/moonstreamapi/moonstreamapi/routes/dashboards.py +++ b/moonstreamapi/moonstreamapi/routes/dashboards.py @@ -1,6 +1,6 @@ import json import logging -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union, cast from uuid import UUID import boto3 # type: ignore @@ -61,17 +61,18 @@ async def add_dashboard_handler( subscriptions_list = bc.search( token=token, journal_id=journal_id, - required_field=[f"type:subscription"], + query="tag:type:subscription", limit=1000, representation="entity", ) # process existing subscriptions with supplied ids - - available_subscriptions_ids: Dict[Union[UUID, str], BugoutSearchResultAsEntity] = { - subscription.entity_id: subscription - for subscription in subscriptions_list.entities - } + available_subscriptions_ids: Dict[Union[UUID, str], BugoutSearchResultAsEntity] + for result in subscriptions_list.results: + entity = cast(BugoutSearchResultAsEntity, result) + entity_url_list = entity.entity_url.split("/") + subscription_id = entity_url_list[len(entity_url_list) - 1] + available_subscriptions_ids[subscription_id] = entity for dashboard_subscription in subscription_settings: if dashboard_subscription.subscription_id in available_subscriptions_ids.keys(): @@ -234,15 +235,17 @@ async def update_dashboard_handler( subscriptions_list = bc.search( token=token, journal_id=journal_id, - required_field=[f"type:subscription"], + query="tag:type:subscription", limit=1000, representation="entity", ) - available_subscriptions_ids: Dict[Union[UUID, str], BugoutSearchResultAsEntity] = { - subscription.entity_id: subscription - for subscription in subscriptions_list.entities - } + available_subscriptions_ids: Dict[Union[UUID, str], BugoutSearchResultAsEntity] + for result in subscriptions_list.results: + entity = cast(BugoutSearchResultAsEntity, result) + entity_url_list = entity.entity_url.split("/") + subscription_id = entity_url_list[len(entity_url_list) - 1] + available_subscriptions_ids[subscription_id] = entity for dashboard_subscription in subscription_settings: if dashboard_subscription.subscription_id in available_subscriptions_ids: @@ -337,12 +340,12 @@ async def get_dashboard_data_links_handler( subscriptions_list = bc.search( token=token, journal_id=journal_id, - required_field=[f"type:subscription"], + query="tag:type:subscription", limit=1000, representation="entity", ) - # filter out dasboards + # filter out dashboards subscriptions_ids = [ subscription_meta["subscription_id"] @@ -351,11 +354,12 @@ async def get_dashboard_data_links_handler( ] ] - dashboard_subscriptions: Dict[Union[UUID, str], BugoutSearchResultAsEntity] = { - subscription.entity_id: subscription - for subscription in subscriptions_list.entities - if str(subscription.entity_id) in subscriptions_ids - } + dashboard_subscriptions: Dict[Union[UUID, str], BugoutSearchResultAsEntity] + for result in subscriptions_list.results: + entity = cast(BugoutSearchResultAsEntity, result) + entity_url_list = entity.entity_url.split("/") + subscription_id = entity_url_list[len(entity_url_list) - 1] + dashboard_subscriptions[subscription_id] = entity # generate s3 links diff --git a/moonstreamapi/moonstreamapi/routes/queries.py b/moonstreamapi/moonstreamapi/routes/queries.py index 9f468065..cc90704e 100644 --- a/moonstreamapi/moonstreamapi/routes/queries.py +++ b/moonstreamapi/moonstreamapi/routes/queries.py @@ -1,40 +1,38 @@ """ The Moonstream queries HTTP API """ -from datetime import datetime import logging +from datetime import datetime from typing import Any, Dict, List, Optional, Tuple, Union from uuid import UUID - -from bugout.data import BugoutResources, BugoutJournalEntryContent, BugoutJournalEntry -from bugout.exceptions import BugoutResponseException -from fastapi import APIRouter, Body, Request import requests # type: ignore +from bugout.data import BugoutJournalEntry, BugoutJournalEntryContent, BugoutResources +from bugout.exceptions import BugoutResponseException +from fastapi import APIRouter, Body, Path, Request from moonstreamdb.blockchain import AvailableBlockchainType from sqlalchemy import text - from .. import data from ..actions import ( + NameNormalizationException, + generate_s3_access_links, get_query_by_name, name_normalization, - NameNormalizationException, query_parameter_hash, - generate_s3_access_links, ) from ..middleware import MoonstreamHTTPException from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_APPLICATION_ID, - MOONSTREAM_CRAWLERS_SERVER_URL, MOONSTREAM_CRAWLERS_SERVER_PORT, + MOONSTREAM_CRAWLERS_SERVER_URL, + MOONSTREAM_QUERIES_JOURNAL_ID, + MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE, MOONSTREAM_S3_QUERIES_BUCKET, MOONSTREAM_S3_QUERIES_BUCKET_PREFIX, - MOONSTREAM_QUERIES_JOURNAL_ID, ) -from ..settings import bugout_client as bc, MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE - +from ..settings import bugout_client as bc logger = logging.getLogger(__name__) @@ -183,7 +181,7 @@ def get_suggested_queries( query = " ".join(filters) try: - queries = bc.search( + queries: Any = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, query=query, @@ -232,7 +230,7 @@ async def get_query_handler( ) # check in templates - + entries: Any try: entries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -390,6 +388,7 @@ async def update_query_data_handler( ) # check in templates + entries: Any try: entries = bc.search( @@ -497,6 +496,7 @@ async def get_access_link_handler( ) # check in templattes + entries: Any try: entries = bc.search( diff --git a/moonstreamapi/moonstreamapi/routes/subscriptions.py b/moonstreamapi/moonstreamapi/routes/subscriptions.py index 734c10c3..e9750c3e 100644 --- a/moonstreamapi/moonstreamapi/routes/subscriptions.py +++ b/moonstreamapi/moonstreamapi/routes/subscriptions.py @@ -275,12 +275,6 @@ async def delete_subscription_handler( abi = deleted_entity.secondary_fields.get("abi") description = deleted_entity.secondary_fields.get("description") - deleted_entity_required_fields = ( - deleted_entity.required_fields - if deleted_entity.required_fields is not None - else [] - ) - return data.SubscriptionResourceData( id=str(deleted_entity.id), user_id=str(user.id), @@ -289,7 +283,7 @@ async def delete_subscription_handler( label=label, abi=abi, description=description, - tags=deleted_entity_required_fields, + tags=deleted_entity.required_fields, subscription_type_id=subscription_type_id, updated_at=deleted_entity.updated_at, created_at=deleted_entity.created_at, @@ -299,8 +293,8 @@ async def delete_subscription_handler( @router.get("/", tags=["subscriptions"], response_model=data.SubscriptionsListResponse) async def get_subscriptions_handler( request: Request, - limit: Optional[int] = Query(10), - offset: Optional[int] = Query(0), + limit: int = Query(10), + offset: int = Query(0), ) -> data.SubscriptionsListResponse: """ Get user's subscriptions. @@ -314,7 +308,7 @@ async def get_subscriptions_handler( user_id=user.id, create_if_not_exist=True, ) - subscriptions_list = bc.search( + subscriptions_list: Any = bc.search( token=token, journal_id=journal_id, query="tag:type:subscription", @@ -423,16 +417,21 @@ async def update_subscriptions_handler( entity_id=subscription_id, ) + update_required_fields = [] + if subscription_entity.required_fields is not None: + update_required_fields = [ + field + for field in subscription_entity.required_fields + if any(key in field for key in MOONSTREAM_ENTITIES_RESERVED_TAGS) + ] + + update_secondary_fields = ( + subscription_entity.secondary_fields + if subscription_entity.secondary_fields is not None + else {} + ) + subscription_type_id = None - - update_required_fields = [ - field - for field in subscription_entity.required_fields - if any(key in field for key in MOONSTREAM_ENTITIES_RESERVED_TAGS) - ] - - update_secondary_fields = subscription_entity.secondary_fields - for field in update_required_fields: if "subscription_type_id" in field: subscription_type_id = field["subscription_type_id"] @@ -496,14 +495,24 @@ async def update_subscriptions_handler( if allowed_required_fields: update_required_fields.extend(allowed_required_fields) + + address = subscription_entity.address + if address is None: + logger.error(f"Lost address at entity {subscription_id} for subscription") + raise MoonstreamHTTPException(status_code=500) + try: subscription = bc.update_entity( token=token, journal_id=journal_id, entity_id=subscription_id, - title=subscription_entity.title, - address=subscription_entity.address, - blockchain=subscription_entity.blockchain, + title=subscription_entity.title + if subscription_entity.title is not None + else "", + address=address, + blockchain=subscription_entity.blockchain + if subscription_entity.blockchain is not None + else "", required_fields=update_required_fields, secondary_fields=update_secondary_fields, ) @@ -516,12 +525,20 @@ async def update_subscriptions_handler( apply_moonworm_tasks, subscription_type_id, json_abi, - subscription.address, + address, ) + subscription_required_fields = ( + subscription.required_fields if subscription.required_fields is not None else {} + ) + subscription_secondary_fields = ( + subscription.secondary_fields + if subscription.secondary_fields is not None + else {} + ) normalized_entity_tags = [ f"{key}:{value}" - for tag in subscription.required_fields + for tag in subscription_required_fields for key, value in tag.items() if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS ] @@ -532,8 +549,8 @@ async def update_subscriptions_handler( address=subscription.address, color=color, label=label, - abi=subscription.secondary_fields.get("abi"), - description=subscription.secondary_fields.get("description"), + abi=subscription_secondary_fields.get("abi"), + description=subscription_secondary_fields.get("description"), tags=normalized_entity_tags, subscription_type_id=subscription_type_id, updated_at=subscription_entity.updated_at, From 1ee2436e7fb4f2fb42d934af95feaa2353397a7a Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 31 Jul 2023 15:40:21 +0000 Subject: [PATCH 07/50] Fix filter out dashboards back --- moonstreamapi/moonstreamapi/routes/dashboards.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moonstreamapi/moonstreamapi/routes/dashboards.py b/moonstreamapi/moonstreamapi/routes/dashboards.py index 594e5d17..cfa96607 100644 --- a/moonstreamapi/moonstreamapi/routes/dashboards.py +++ b/moonstreamapi/moonstreamapi/routes/dashboards.py @@ -359,7 +359,8 @@ async def get_dashboard_data_links_handler( entity = cast(BugoutSearchResultAsEntity, result) entity_url_list = entity.entity_url.split("/") subscription_id = entity_url_list[len(entity_url_list) - 1] - dashboard_subscriptions[subscription_id] = entity + if str(subscription_id) in subscriptions_ids: + dashboard_subscriptions[subscription_id] = entity # generate s3 links From 1e01a89b1b90222b18ca21fb19ed729faa4b4b7c Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 31 Jul 2023 16:00:59 +0000 Subject: [PATCH 08/50] mypy fixes --- .../moonstreamapi/routes/dashboards.py | 10 +++--- .../moonstreamapi/routes/subscriptions.py | 34 +++++++++++++------ 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/moonstreamapi/moonstreamapi/routes/dashboards.py b/moonstreamapi/moonstreamapi/routes/dashboards.py index cfa96607..91204094 100644 --- a/moonstreamapi/moonstreamapi/routes/dashboards.py +++ b/moonstreamapi/moonstreamapi/routes/dashboards.py @@ -262,12 +262,10 @@ async def update_dashboard_handler( status_code=404, detail=f"Error on dashboard resource {dashboard_subscription.subscription_id} does not have an abi", ) - - abi = json.loads( - available_subscriptions_ids[ - dashboard_subscription.subscription_id - ].secondary_fields.get("abi") - ) + abi_raw = available_subscriptions_ids[ + dashboard_subscription.subscription_id + ].secondary_fields.get("abi") + abi = json.loads(abi_raw if abi_raw is not None else "") actions.dashboards_abi_validation(dashboard_subscription, abi) diff --git a/moonstreamapi/moonstreamapi/routes/subscriptions.py b/moonstreamapi/moonstreamapi/routes/subscriptions.py index e9750c3e..aa1132c4 100644 --- a/moonstreamapi/moonstreamapi/routes/subscriptions.py +++ b/moonstreamapi/moonstreamapi/routes/subscriptions.py @@ -252,7 +252,11 @@ async def delete_subscription_handler( detail="Internal error", ) - tags = deleted_entity.required_fields + tags_raw = ( + deleted_entity.required_fields + if deleted_entity.required_fields is not None + else {} + ) subscription_type_id = None color = None @@ -260,16 +264,20 @@ async def delete_subscription_handler( abi = None description = None - if tags is not None: - for tag in tags: - if "subscription_type_id" in tag: - subscription_type_id = tag["subscription_type_id"] + for tag in tags_raw: + if "subscription_type_id" in tag: + subscription_type_id = tag["subscription_type_id"] + if "color" in tag: + color = tag["color"] + if "label" in tag: + label = tag["label"] - if "color" in tag: - color = tag["color"] - - if "label" in tag: - label = tag["label"] + normalized_entity_tags = [ + f"{key}:{value}" + for tag in tags_raw + for key, value in tag.items() + if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS + ] if deleted_entity.secondary_fields is not None: abi = deleted_entity.secondary_fields.get("abi") @@ -283,7 +291,7 @@ async def delete_subscription_handler( label=label, abi=abi, description=description, - tags=deleted_entity.required_fields, + tags=normalized_entity_tags, subscription_type_id=subscription_type_id, updated_at=deleted_entity.updated_at, created_at=deleted_entity.created_at, @@ -594,6 +602,10 @@ async def get_subscription_abi_handler( logger.error(f"Error get subscriptions for user ({user}), error: {str(e)}") raise MoonstreamHTTPException(status_code=500, internal_error=e) + if subscription_resource.secondary_fields is None: + raise MoonstreamHTTPException( + status_code=500, detail=f"Malformed subscription entity {subscription_id}" + ) if "abi" not in subscription_resource.secondary_fields.keys(): raise MoonstreamHTTPException(status_code=404, detail="Abi not found") From 92dbfe55764c22a8f735b71840f8b449d8cbb1ac Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 1 Aug 2023 08:58:36 +0000 Subject: [PATCH 09/50] Mooncrawl entity migration to spire --- crawlers/mooncrawl/mooncrawl/actions.py | 2 +- crawlers/mooncrawl/mooncrawl/api.py | 44 ++++++++-------- .../mooncrawl/leaderboards_generator/cli.py | 11 ++-- .../mooncrawl/moonworm_crawler/crawler.py | 12 +++-- crawlers/mooncrawl/mooncrawl/settings.py | 11 ---- .../mooncrawl/stats_worker/dashboard.py | 51 +++++++++++-------- crawlers/mooncrawl/sample.env | 3 -- crawlers/mooncrawl/setup.py | 3 +- 8 files changed, 64 insertions(+), 73 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/actions.py b/crawlers/mooncrawl/mooncrawl/actions.py index 7372a547..aaa05693 100644 --- a/crawlers/mooncrawl/mooncrawl/actions.py +++ b/crawlers/mooncrawl/mooncrawl/actions.py @@ -82,7 +82,7 @@ def get_entity_subscription_collection_id( resource_type: str, token: Union[uuid.UUID, str], user_id: uuid.UUID, -) -> Optional[str]: +) -> str: """ Get collection_id from brood resources. If collection not exist and create_if_not_exist is True """ diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index 08bee314..66b1dc5e 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -9,41 +9,39 @@ from typing import Any, Dict, List from uuid import UUID import boto3 # type: ignore -from bugout.data import BugoutResource -from entity.data import EntityResponse # type: ignore +from bugout.data import BugoutJournalEntity, BugoutResource from fastapi import BackgroundTasks, FastAPI from fastapi.middleware.cors import CORSMiddleware from moonstreamdb.blockchain import ( AvailableBlockchainType, - get_label_model, get_block_model, + get_label_model, get_transaction_model, ) - from sqlalchemy import text -from .actions import ( - generate_s3_access_links, - query_parameter_hash, - get_entity_subscription_collection_id, - EntityCollectionNotFoundException, -) from . import data +from .actions import ( + EntityCollectionNotFoundException, + generate_s3_access_links, + get_entity_subscription_collection_id, + query_parameter_hash, +) from .middleware import MoonstreamHTTPException from .settings import ( - BUGOUT_RESOURCE_TYPE_SUBSCRIPTION, BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, - MOONSTREAM_ADMIN_ACCESS_TOKEN, + BUGOUT_RESOURCE_TYPE_SUBSCRIPTION, DOCS_TARGET_PATH, + LINKS_EXPIRATION_TIME, + MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_S3_QUERIES_BUCKET, MOONSTREAM_S3_QUERIES_BUCKET_PREFIX, + MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, NB_CONTROLLER_ACCESS_ID, ORIGINS, - LINKS_EXPIRATION_TIME, - MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, ) -from .settings import bugout_client as bc, entity_client as ec +from .settings import bugout_client as bc from .stats_worker import dashboard, queries from .version import MOONCRAWL_VERSION @@ -115,12 +113,11 @@ async def status_handler( ) try: - collection_id = get_entity_subscription_collection_id( + journal_id = get_entity_subscription_collection_id( resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, token=MOONSTREAM_ADMIN_ACCESS_TOKEN, user_id=UUID(stats_update.user_id), ) - except EntityCollectionNotFoundException as e: raise MoonstreamHTTPException( status_code=404, @@ -136,20 +133,19 @@ async def status_handler( s3_client = boto3.client("s3") - subscription_by_id: Dict[str, EntityResponse] = {} + subscription_by_id: Dict[str, BugoutJournalEntity] = {} for dashboard_subscription_filters in dashboard_resource.resource_data[ "subscription_settings" ]: # get subscription by id - - subscription: EntityResponse = ec.get_entity( + subscription: BugoutJournalEntity = bc.get_entity( token=stats_update.token, - collection_id=collection_id, + journal_id=journal_id, entity_id=dashboard_subscription_filters["subscription_id"], ) - subscription_by_id[str(subscription.entity_id)] = subscription + subscription_by_id[str(subscription.id)] = subscription try: background_tasks.add_task( @@ -182,7 +178,7 @@ async def status_handler( subscriprions_type = reqired_field["subscription_type_id"] for timescale in stats_update.timescales: - presigned_urls_response[subscription_entity.entity_id] = {} + presigned_urls_response[subscription_entity.id] = {} try: result_key = f"{MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX}/{dashboard.blockchain_by_subscription_id[subscriprions_type]}/contracts_data/{subscription_entity.address}/{stats_update.dashboard_id}/v1/{timescale}.json" @@ -201,7 +197,7 @@ async def status_handler( HttpMethod="GET", ) - presigned_urls_response[subscription_entity.entity_id][timescale] = { + presigned_urls_response[subscription_entity.id][timescale] = { "url": stats_presigned_url, "headers": { "If-Modified-Since": ( diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py index fdffa077..5c9f9da3 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -2,11 +2,11 @@ import argparse import json import logging import os -from typing import Any, Dict +from typing import cast, List import uuid import requests # type: ignore - +from bugout.data import BugoutSearchResult from .utils import get_results_for_moonstream_query from ..settings import ( @@ -49,17 +49,18 @@ def handle_leaderboards(args: argparse.Namespace) -> None: limit=100, timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) + leaderboards_results = cast(List[BugoutSearchResult], leaderboards.results) except Exception as e: logger.error(f"Could not get leaderboards from journal: {e}") return - if len(leaderboards.results) == 0: + if len(leaderboards_results) == 0: logger.error("No leaderboard found") return - logger.info(f"Found {len(leaderboards.results)} leaderboards") + logger.info(f"Found {len(leaderboards_results)} leaderboards") - for leaderboard in leaderboards.results: + for leaderboard in leaderboards_results: logger.info( f"Processing leaderboard: {leaderboard.title} with id: {[tag for tag in leaderboard.tags if tag.startswith('leaderboard_id')]}" ) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index 0c7dfd54..ff1af5ca 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -195,7 +195,7 @@ def get_crawl_job_entries( query += f" created_at:>={created_at_filter}" current_offset = 0 - entries = [] + entries: List[BugoutSearchResult] = [] while True: search_result = bugout_client.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -205,10 +205,11 @@ def get_crawl_job_entries( limit=limit, timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) - entries.extend(search_result.results) + search_results = cast(List[BugoutSearchResult], search_result.results) + entries.extend(search_results) # if len(entries) >= search_result.total_results: - if len(search_result.results) == 0: + if len(search_results) == 0: break current_offset += limit return entries @@ -402,8 +403,9 @@ def _get_heartbeat_entry_id( limit=1, timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) - if entries.results: - return entries.results[0].entry_url.split("/")[-1] + search_results = cast(List[BugoutSearchResult], entries.results) + if search_results: + return search_results[0].entry_url.split("/")[-1] else: logger.info(f"No {crawler_type} heartbeat entry found, creating one") entry = bugout_client.create_entry( diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index c846a8c9..efecd88b 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -3,10 +3,8 @@ from typing import Dict, Optional from uuid import UUID from bugout.app import Bugout -from entity.client import Entity # type: ignore from moonstreamdb.blockchain import AvailableBlockchainType - # Bugout BUGOUT_BROOD_URL = os.environ.get("BUGOUT_BROOD_URL", "https://auth.bugout.dev") BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev") @@ -14,15 +12,6 @@ BUGOUT_SPIRE_URL = os.environ.get("BUGOUT_SPIRE_URL", "https://spire.bugout.dev" bugout_client = Bugout(brood_api_url=BUGOUT_BROOD_URL, spire_api_url=BUGOUT_SPIRE_URL) -# Entity - - -MOONSTREAM_ENTITY_URL = os.environ.get("MOONSTREAM_ENTITY_URL", "") -if MOONSTREAM_ENTITY_URL == "": - raise ValueError("MOONSTREAM_ENTITY_URL environment variable must be set") - -entity_client = Entity(MOONSTREAM_ENTITY_URL) - MOONSTREAM_API_URL = os.environ.get("MOONSTREAM_API_URL", "https://api.moonstream.to") MOONSTREAM_ENGINE_URL = os.environ.get( "MOONSTREAM_ENGINE_URL", "https://engineapi.moonstream.to" diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 2a223bdd..e21dee68 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -6,21 +6,25 @@ import hashlib import json import logging import time -import traceback from datetime import datetime, timedelta from enum import Enum -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, cast, Dict, List, Optional, Union from uuid import UUID import boto3 # type: ignore -from bugout.data import BugoutResource, BugoutResources -from entity.data import EntityResponse, EntityCollectionResponse # type: ignore +from bugout.data import ( + BugoutJournalEntity, + BugoutResource, + BugoutResources, + BugoutSearchResultAsEntity, +) from moonstreamdb.blockchain import ( AvailableBlockchainType, get_label_model, get_transaction_model, ) -from sqlalchemy import and_, cast, distinct, extract, func, text +from sqlalchemy import and_, distinct, extract, func, text +from sqlalchemy import cast as sqlalchemy_cast from sqlalchemy.orm import Session from sqlalchemy.sql.operators import in_op from web3 import Web3 @@ -29,14 +33,14 @@ from ..blockchain import connect from ..db import yield_db_read_only_session_ctx from ..reporter import reporter from ..settings import ( + BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, CRAWLER_LABEL, MOONSTREAM_ADMIN_ACCESS_TOKEN, + MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, NB_CONTROLLER_ACCESS_ID, - MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, - BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION, ) -from ..settings import bugout_client as bc, entity_client as ec +from ..settings import bugout_client as bc logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -157,11 +161,13 @@ def generate_data( .filter(in_op(label_model.label_data["name"].astext, functions)) .filter( label_model.block_timestamp - >= cast(extract("epoch", start), label_model.block_timestamp.type) + >= sqlalchemy_cast( + extract("epoch", start), label_model.block_timestamp.type + ) ) .filter( label_model.block_timestamp - < cast( + < sqlalchemy_cast( extract("epoch", (start + timescales_delta[timescale]["timedelta"])), label_model.block_timestamp.type, ) @@ -652,25 +658,26 @@ def stats_generate_handler(args: argparse.Namespace): address_dashboard_id_subscription_id_tree: Dict[str, Any] = {} - for user_id, collection_id in user_collection_by_id.items(): + for user_id, journal_id in user_collection_by_id.items(): # request all subscriptions for user - user_subscriptions: EntityCollectionResponse = ec.search_entities( + user_subscriptions = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - collection_id=collection_id, - required_field=[ - "subscription_type_id:{}".format( - subscription_id_by_blockchain[args.blockchain] - ) - ], + journal_id=journal_id, + query=f"tag:subscription_type_id:{subscription_id_by_blockchain[args.blockchain]}", + representation="entity", ) + user_subscriptions_results = cast( + List[BugoutSearchResultAsEntity], user_subscriptions.results + ) logger.info( - f"Amount of user subscriptions: {len(user_subscriptions.entities)}" + f"Amount of user subscriptions: {len(user_subscriptions_results)}" ) - for subscription in user_subscriptions.entities: - subscription_id = str(subscription.entity_id) + for subscription in user_subscriptions_results: + entity_url_list = subscription.entity_url.split("/") + subscription_id = entity_url_list[len(entity_url_list) - 1] if subscription_id not in dashboards_by_subscription: logger.info( @@ -1014,7 +1021,7 @@ def stats_generate_handler(args: argparse.Namespace): def stats_generate_api_task( timescales: List[str], dashboard: BugoutResource, - subscription_by_id: Dict[str, EntityResponse], + subscription_by_id: Dict[str, BugoutJournalEntity], access_id: Optional[UUID] = None, ): """ diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index c1db68a4..a02901a5 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -4,9 +4,6 @@ export BUGOUT_SPIRE_URL="https://spire.bugout.dev" export HUMBUG_REPORTER_CRAWLERS_TOKEN="" -# Entity environment variables -export MOONSTREAM_ENTITY_URL="https://api.moonstream.to/entity" - # Engine environment variables export MOONSTREAM_ENGINE_URL="https://engineapi.moonstream.to" diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 95751fe2..2df7f7db 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -34,12 +34,11 @@ setup( zip_safe=False, install_requires=[ "boto3", - "bugout>=0.2.8", + "bugout>=0.2.12", "chardet", "fastapi", "moonstreamdb>=0.3.4", "moonstream>=0.1.1", - "moonstream-entity>=0.0.5", "moonworm[moonstream]>=0.6.2", "humbug", "pydantic==1.9.2", From e854978459c4f119ad374ecfba9b88c04cc5ae71 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 1 Aug 2023 11:45:24 +0000 Subject: [PATCH 10/50] Fixed validation of new call requests set not subscriptable --- engineapi/engineapi/contracts_actions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index 5f859773..8e7ad875 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -66,7 +66,6 @@ def validate_method_and_params( Web3.toChecksumAddress(parameters["signer"]) except: raise InvalidAddressFormat("Parameter signer must be a valid address") - required_params["amount"] = str(required_params["amount"]) else: raise ValueError(f"Unknown contract type {contract_type}") From 29e512538e58c371b9eb2c5c806dcbcc95135852 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 2 Aug 2023 15:49:16 +0300 Subject: [PATCH 11/50] Fix list subscriptions. --- moonstreamapi/moonstreamapi/routes/subscriptions.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/moonstreamapi/moonstreamapi/routes/subscriptions.py b/moonstreamapi/moonstreamapi/routes/subscriptions.py index aa1132c4..9e8ae77c 100644 --- a/moonstreamapi/moonstreamapi/routes/subscriptions.py +++ b/moonstreamapi/moonstreamapi/routes/subscriptions.py @@ -5,9 +5,9 @@ import hashlib import json import logging from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union, cast -from bugout.data import BugoutSearchResult +from bugout.data import BugoutSearchResult, BugoutSearchResultAsEntity from bugout.exceptions import BugoutResponseException from fastapi import APIRouter, BackgroundTasks, Depends, Form, Path, Query, Request from moonstreamdb.blockchain import AvailableBlockchainType @@ -322,6 +322,7 @@ async def get_subscriptions_handler( query="tag:type:subscription", limit=limit, offset=offset, + representation="entity", ) except EntityJournalNotFoundException as e: @@ -339,7 +340,11 @@ async def get_subscriptions_handler( subscriptions = [] - for subscription in subscriptions_list.entities: + user_subscriptions_results = cast( + List[BugoutSearchResultAsEntity], subscriptions_list.results + ) + + for subscription in user_subscriptions_results: tags = subscription.required_fields label, color, subscription_type_id = None, None, None @@ -363,7 +368,7 @@ async def get_subscriptions_handler( subscriptions.append( data.SubscriptionResourceData( - id=str(subscription.id), + id=str(subscription.entity_url.split("/")[-1]), user_id=str(user.id), address=subscription.address, color=color, From 179370affb82054e0c794ee6c320ee412b4e69db Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Aug 2023 07:27:02 +0000 Subject: [PATCH 12/50] Replaced Any with cast --- moonstreamapi/moonstreamapi/routes/queries.py | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/moonstreamapi/moonstreamapi/routes/queries.py b/moonstreamapi/moonstreamapi/routes/queries.py index cc90704e..e0f5f035 100644 --- a/moonstreamapi/moonstreamapi/routes/queries.py +++ b/moonstreamapi/moonstreamapi/routes/queries.py @@ -3,11 +3,16 @@ The Moonstream queries HTTP API """ import logging from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union, cast from uuid import UUID import requests # type: ignore -from bugout.data import BugoutJournalEntry, BugoutJournalEntryContent, BugoutResources +from bugout.data import ( + BugoutJournalEntry, + BugoutJournalEntryContent, + BugoutResources, + BugoutSearchResult, +) from bugout.exceptions import BugoutResponseException from fastapi import APIRouter, Body, Path, Request from moonstreamdb.blockchain import AvailableBlockchainType @@ -181,7 +186,7 @@ def get_suggested_queries( query = " ".join(filters) try: - queries: Any = bc.search( + queries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, query=query, @@ -197,7 +202,9 @@ def get_suggested_queries( interfaces: Dict[str, Any] = {} - for entry in queries.results: + queries_results = cast(List[BugoutSearchResult], queries.results) + + for entry in queries_results: for tag in entry.tags: if tag.startswith("interface:"): interface = tag.split(":")[1] @@ -208,7 +215,7 @@ def get_suggested_queries( interfaces[interface].append(entry) return data.SuggestedQueriesResponse( - queries=queries.results, + queries=queries_results, interfaces=interfaces, ) @@ -230,7 +237,6 @@ async def get_query_handler( ) # check in templates - entries: Any try: entries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -275,9 +281,11 @@ async def get_query_handler( status_code=403, detail="Query not approved yet." ) else: - query_id = entries.results[0].entry_url.split("/")[-1] + entries_results = cast(List[BugoutSearchResult], entries.results) + query_id = entries_results[0].entry_url.split("/")[-1] - entry = entries.results[0] + entries_results = cast(List[BugoutSearchResult], entries.results) + entry = entries_results[0] try: if entry.content is None: @@ -388,8 +396,6 @@ async def update_query_data_handler( ) # check in templates - entries: Any - try: entries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -434,14 +440,16 @@ async def update_query_data_handler( status_code=403, detail="Query not approved yet." ) else: - query_id = entries.results[0].entry_url.split("/")[-1] + entries_results = cast(List[BugoutSearchResult], entries.results) + query_id = entries_results[0].entry_url.split("/")[-1] s3_response = None - if entries.results[0].content: - content = entries.results[0].content + entries_results = cast(List[BugoutSearchResult], entries.results) + if entries_results[0].content: + content = entries_results[0].content - tags = entries.results[0].tags + tags = entries_results[0].tags file_type = "json" @@ -496,8 +504,6 @@ async def get_access_link_handler( ) # check in templattes - entries: Any - try: entries = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -540,13 +546,14 @@ async def get_access_link_handler( status_code=403, detail="Query not approved yet." ) + entries_results = cast(List[BugoutSearchResult], entries.results) try: s3_response = None - if entries.results[0].content: + if entries_results[0].content: passed_params = dict(request_update.params) - tags = entries.results[0].tags + tags = entries_results[0].tags file_type = "json" From 3b265ad15dcd0eb9b4511a5107ae75dfebf7c76f Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Aug 2023 09:17:05 +0000 Subject: [PATCH 13/50] Migration Call request types and Metatx holders --- ...f_call_request_types_and_metatx_holders.py | 162 ++++++++++++++++++ engineapi/engineapi/contracts_actions.py | 1 - engineapi/engineapi/models.py | 128 ++++++++++++-- 3 files changed, 276 insertions(+), 15 deletions(-) create mode 100644 engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py diff --git a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py new file mode 100644 index 00000000..2d3ef981 --- /dev/null +++ b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py @@ -0,0 +1,162 @@ +"""Call request types and Metatx holders + +Revision ID: b4257b10daaf +Revises: dedd8a7d0624 +Create Date: 2023-08-02 18:28:14.724453 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b4257b10daaf' +down_revision = 'dedd8a7d0624' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + + # Blockchains + op.create_table('blockchains', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('name', sa.VARCHAR(length=128), nullable=False), + sa.Column('chain_id', sa.Integer(), nullable=False), + sa.Column('testnet', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_blockchains')), + sa.UniqueConstraint('id', name=op.f('uq_blockchains_id')) + ) + + op.create_index(op.f('ix_blockchains_chain_id'), 'blockchains', ['chain_id'], unique=False) + op.create_index(op.f('ix_blockchains_name'), 'blockchains', ['name'], unique=True) + + op.add_column('registered_contracts', sa.Column('blockchain_id', sa.UUID(), nullable=True)) + op.create_foreign_key(op.f('fk_registered_contracts_blockchain_id_blockchains'), 'registered_contracts', 'blockchains', ['blockchain_id'], ['id'], ondelete='CASCADE') + + # Manual - Start + op.execute("INSERT INTO blockchains (name, chain_id, testnet) VALUES ('polygon', 137, FALSE);") + op.execute("INSERT INTO blockchains (name, chain_id, testnet) VALUES ('mumbai', 80001, TRUE);") + op.execute("UPDATE registered_contracts SET blockchain_id = (SELECT id FROM blockchains WHERE blockchains.name = contracts.blockchain);") + op.alter_column("registered_contracts", "blockchain_id", nullable=False) + # Manual - End + + op.drop_constraint('uq_registered_contracts_blockchain', 'registered_contracts', type_='unique') + op.drop_index('ix_registered_contracts_blockchain', table_name='registered_contracts') + op.drop_column('registered_contracts', 'blockchain') + + # Types + op.create_table('call_request_types', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('request_type', sa.VARCHAR(length=128), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_call_request_types')), + sa.UniqueConstraint('id', name=op.f('uq_call_request_types_id')) + ) + + op.create_index(op.f('ix_call_request_types_request_type'), 'call_request_types', ['request_type'], unique=True) + + op.add_column('call_requests', sa.Column('call_request_type_id', sa.UUID(), nullable=True)) + op.create_foreign_key(op.f('fk_call_requests_call_request_id_call_request_types'), 'call_requests', 'call_request_types', ['call_request_id'], ['id'], ondelete='CASCADE') + + # Manual - Start + op.execute("INSERT INTO call_request_types (request_type) VALUES ('raw');") + op.execute("INSERT INTO call_request_types (request_type) VALUES ('dropper-v0.2.0');") + op.execute("UPDATE call_requests SET request_type = (SELECT id FROM call_request_types INNER JOIN registered_contracts ON call_requests.registered_contract_id = registered_contracts.id WHERE call_request_types.request_type = registered_contracts.contract_type);") + op.alter_column("call_requests", "call_request_type_id", nullable=False) + # Manual - End + + op.drop_index('ix_registered_contracts_contract_type', table_name='registered_contracts') + op.drop_column('registered_contracts', 'contract_type') + + # Holders + op.create_table('metatx_holders', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_metatx_holders')), + sa.UniqueConstraint('id', name=op.f('uq_metatx_holders_id')) + ) + + op.add_column('call_requests', sa.Column('metatx_holder_id', sa.UUID(), nullable=True)) + op.create_foreign_key(op.f('fk_call_requests_metatx_holder_id_metatx_holders'), 'call_requests', 'metatx_holders', ['metatx_holder_id'], ['id'], ondelete='CASCADE') + op.add_column('registered_contracts', sa.Column('metatx_holder_id', sa.UUID(), nullable=True)) + op.create_foreign_key(op.f('fk_registered_contracts_metatx_holder_id_metatx_holders'), 'registered_contracts', 'metatx_holders', ['metatx_holder_id'], ['id'], ondelete='CASCADE') + + # Manual - Start + op.execute("INSERT INTO metatx_holders (id) SELECT DISTINCT moonstream_user_id FROM registered_contracts ON CONFLICT (id) DO NOTHING;") + op.execute("INSERT INTO metatx_holders (id) SELECT DISTINCT moonstream_user_id FROM call_requests ON CONFLICT (id) DO NOTHING;") + op.execute("UPDATE registered_contracts SET metatx_holder_id = moonstream_user_id;") + op.execute("UPDATE call_requests SET metatx_holder_id = moonstream_user_id;") + op.alter_column("call_requests", "metatx_holder_id", nullable=False) + op.alter_column("registered_contracts", "metatx_holder_id", nullable=False) + # Manual - End + + op.drop_index('ix_call_requests_moonstream_user_id', table_name='call_requests') + op.drop_column('call_requests', 'moonstream_user_id') + op.drop_index('ix_registered_contracts_moonstream_user_id', table_name='registered_contracts') + op.drop_column('registered_contracts', 'moonstream_user_id') + + # Other + op.create_unique_constraint(op.f('uq_registered_contracts_blockchain_id'), 'registered_contracts', ['blockchain_id', 'metatx_holder_id', 'address']) + + op.create_unique_constraint(op.f('uq_call_requests_id'), 'call_requests', ['id']) + op.create_unique_constraint(op.f('uq_registered_contracts_id'), 'registered_contracts', ['id']) + op.create_unique_constraint(op.f('uq_leaderboard_scores_id'), 'leaderboard_scores', ['id']) + op.create_unique_constraint(op.f('uq_leaderboards_id'), 'leaderboards', ['id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + + # Blockchains + op.add_column('registered_contracts', sa.Column('blockchain', sa.VARCHAR(length=128), autoincrement=False, nullable=False)) + op.create_index('ix_registered_contracts_blockchain', 'registered_contracts', ['blockchain'], unique=False) + + # Manual + # Copy blockchain values + # ... + + op.drop_constraint(op.f('fk_registered_contracts_blockchain_id_blockchains'), 'registered_contracts', type_='foreignkey') + op.drop_constraint(op.f('uq_registered_contracts_blockchain_id'), 'registered_contracts', type_='unique') + op.drop_column('registered_contracts', 'blockchain_id') + + op.drop_index(op.f('ix_blockchains_name'), table_name='blockchains') + op.drop_index(op.f('ix_blockchains_chain_id'), table_name='blockchains') + op.drop_table('blockchains') + + # Types + op.add_column('registered_contracts', sa.Column('contract_type', sa.VARCHAR(length=128), autoincrement=False, nullable=False)) + op.create_index('ix_registered_contracts_contract_type', 'registered_contracts', ['contract_type'], unique=False) + + # Manual + # Copy type values + # ... + + op.drop_constraint(op.f('fk_call_requests_call_request_id_call_request_types'), 'call_requests', type_='foreignkey') + op.drop_column('call_requests', 'call_request_type_id') + + op.drop_index(op.f('ix_call_request_types_request_type'), table_name='call_request_types') + op.drop_table('call_request_types') + + # Holders + op.add_column('registered_contracts', sa.Column('moonstream_user_id', sa.UUID(), autoincrement=False, nullable=False)) + op.create_index('ix_registered_contracts_moonstream_user_id', 'registered_contracts', ['moonstream_user_id'], unique=False) + op.add_column('call_requests', sa.Column('moonstream_user_id', sa.UUID(), autoincrement=False, nullable=False)) + op.create_index('ix_call_requests_moonstream_user_id', 'call_requests', ['moonstream_user_id'], unique=False) + + # Manual + # Copy moonstream_user_ids + # ... + + op.drop_constraint(op.f('fk_registered_contracts_metatx_holder_id_metatx_holders'), 'registered_contracts', type_='foreignkey') + op.drop_column('registered_contracts', 'metatx_holder_id') + op.drop_constraint(op.f('fk_call_requests_metatx_holder_id_metatx_holders'), 'call_requests', type_='foreignkey') + op.drop_column('call_requests', 'metatx_holder_id') + + op.drop_table('metatx_holders') + + # Other + op.create_unique_constraint('uq_registered_contracts_blockchain', 'registered_contracts', ['blockchain', 'moonstream_user_id', 'address', 'contract_type']) + + # ### end Alembic commands ### diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index 5f859773..8e7ad875 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -66,7 +66,6 @@ def validate_method_and_params( Web3.toChecksumAddress(parameters["signer"]) except: raise InvalidAddressFormat("Parameter signer must be a valid address") - required_params["amount"] = str(required_params["amount"]) else: raise ValueError(f"Unknown contract type {contract_type}") diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index 2518be42..d98136cd 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -11,9 +11,11 @@ from sqlalchemy import ( MetaData, String, UniqueConstraint, + Integer, ) from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.ext.compiler import compiles +from sqlalchemy.orm import relationship from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import and_, expression @@ -155,14 +157,86 @@ class DropperClaimant(Base): # type: ignore ) +class CallRequestType(Base): # type: ignore + """ + CallRequestType contains versions of call requests like: + raw or dropper-v0.2.0. + """ + + __tablename__ = "call_request_types" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + ) + request_type = Column(VARCHAR(128), nullable=False, unique=True, index=True) + + call_requests = relationship( + "CallRequest", + back_populates="call_request_type", + cascade="all, delete, delete-orphan", + ) + + +class MetatxHolder(Base): # type: ignore + """ + MetatxHolder represents id of user from bugout authorization. + """ + + __tablename__ = "metatx_holders" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + ) + + created_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + registered_contracts = relationship( + "RegisteredContract", + back_populates="metatx_holder", + cascade="all, delete, delete-orphan", + ) + call_requests = relationship( + "CallRequest", + back_populates="metatx_holder", + cascade="all, delete, delete-orphan", + ) + + +class Blockchain(Base): # type: ignore + __tablename__ = "blockchains" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + ) + name = Column(VARCHAR(128), nullable=False, index=True, unique=True) + chain_id = Column(Integer, nullable=False, index=True, unique=False) + testnet = Column(Boolean, default=False, nullable=False) + + registered_contracts = relationship( + "RegisteredContract", + back_populates="blockchain", + cascade="all, delete, delete-orphan", + ) + + class RegisteredContract(Base): # type: ignore __tablename__ = "registered_contracts" __table_args__ = ( UniqueConstraint( - "blockchain", - "moonstream_user_id", + "blockchain_id", + "metatx_holder_id", "address", - "contract_type", ), ) @@ -172,14 +246,21 @@ class RegisteredContract(Base): # type: ignore default=uuid.uuid4, unique=True, ) - blockchain = Column(VARCHAR(128), nullable=False, index=True) + metatx_holder_id = Column( + UUID(as_uuid=True), + ForeignKey("metatx_holders.id", ondelete="CASCADE"), + nullable=False, + ) + blockchain_id = Column( + UUID(as_uuid=True), + ForeignKey("blockchains.id", ondelete="CASCADE"), + nullable=False, + ) + address = Column(VARCHAR(256), nullable=False, index=True) - contract_type = Column(VARCHAR(128), nullable=False, index=True) title = Column(VARCHAR(128), nullable=False) description = Column(String, nullable=True) image_uri = Column(String, nullable=True) - # User ID of the Moonstream user who registered this contract. - moonstream_user_id = Column(UUID(as_uuid=True), nullable=False, index=True) created_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False @@ -191,6 +272,15 @@ class RegisteredContract(Base): # type: ignore nullable=False, ) + call_requests = relationship( + "CallRequest", + back_populates="registered_contract", + cascade="all, delete, delete-orphan", + ) + + metatx_holder = relationship("MetatxHolder", back_populates="registered_contracts") + blockchain = relationship("Blockchain", back_populates="registered_contracts") + class CallRequest(Base): __tablename__ = "call_requests" @@ -202,19 +292,23 @@ class CallRequest(Base): unique=True, nullable=False, ) - registered_contract_id = Column( UUID(as_uuid=True), ForeignKey("registered_contracts.id", ondelete="CASCADE"), nullable=False, ) + call_request_type_id = Column( + UUID(as_uuid=True), + ForeignKey("call_request_types.id", ondelete="CASCADE"), + nullable=False, + ) + metatx_holder_id = Column( + UUID(as_uuid=True), + ForeignKey("metatx_holders.id", ondelete="CASCADE"), + nullable=False, + ) + caller = Column(VARCHAR(256), nullable=False, index=True) - # User ID of the Moonstream user who requested this call. - # For now, this duplicates the moonstream_user_id in the registered_contracts table. Nevertheless, - # we keep this column here for auditing purposes. In the future, we will add a group_id column to - # the registered_contracts table, and this column will be used to track the user from that group - # who made each call request. - moonstream_user_id = Column(UUID(as_uuid=True), nullable=False, index=True) method = Column(String, nullable=False, index=True) # TODO(zomglings): Should we conditional indices on parameters depending on the contract type? parameters = Column(JSONB, nullable=False) @@ -231,6 +325,12 @@ class CallRequest(Base): nullable=False, ) + registered_contract = relationship( + "RegisteredContract", back_populates="call_requests" + ) + call_request_type = relationship("CallRequestType", back_populates="call_requests") + metatx_holder = relationship("MetatxHolder", back_populates="call_requests") + class Leaderboard(Base): # type: ignore __tablename__ = "leaderboards" From 39a64cf454c584a82efee189060707a8874f2d20 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Aug 2023 09:52:50 +0000 Subject: [PATCH 14/50] Working migration --- ...af_call_request_types_and_metatx_holders.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py index 2d3ef981..8ff6ba51 100644 --- a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py +++ b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py @@ -5,6 +5,8 @@ Revises: dedd8a7d0624 Create Date: 2023-08-02 18:28:14.724453 """ +import uuid + from alembic import op import sqlalchemy as sa @@ -36,9 +38,9 @@ def upgrade(): op.create_foreign_key(op.f('fk_registered_contracts_blockchain_id_blockchains'), 'registered_contracts', 'blockchains', ['blockchain_id'], ['id'], ondelete='CASCADE') # Manual - Start - op.execute("INSERT INTO blockchains (name, chain_id, testnet) VALUES ('polygon', 137, FALSE);") - op.execute("INSERT INTO blockchains (name, chain_id, testnet) VALUES ('mumbai', 80001, TRUE);") - op.execute("UPDATE registered_contracts SET blockchain_id = (SELECT id FROM blockchains WHERE blockchains.name = contracts.blockchain);") + op.execute(f"INSERT INTO blockchains (id, name, chain_id, testnet) VALUES ('{str(uuid.uuid4())}', 'polygon', 137, FALSE);") + op.execute(f"INSERT INTO blockchains (id, name, chain_id, testnet) VALUES ('{str(uuid.uuid4())}', 'mumbai', 80001, TRUE);") + op.execute("UPDATE registered_contracts SET blockchain_id = (SELECT id FROM blockchains WHERE blockchains.name = registered_contracts.blockchain);") op.alter_column("registered_contracts", "blockchain_id", nullable=False) # Manual - End @@ -57,12 +59,12 @@ def upgrade(): op.create_index(op.f('ix_call_request_types_request_type'), 'call_request_types', ['request_type'], unique=True) op.add_column('call_requests', sa.Column('call_request_type_id', sa.UUID(), nullable=True)) - op.create_foreign_key(op.f('fk_call_requests_call_request_id_call_request_types'), 'call_requests', 'call_request_types', ['call_request_id'], ['id'], ondelete='CASCADE') + op.create_foreign_key(op.f('fk_call_requests_call_request_type_id_call_request_types'), 'call_requests', 'call_request_types', ['call_request_type_id'], ['id'], ondelete='CASCADE') # Manual - Start - op.execute("INSERT INTO call_request_types (request_type) VALUES ('raw');") - op.execute("INSERT INTO call_request_types (request_type) VALUES ('dropper-v0.2.0');") - op.execute("UPDATE call_requests SET request_type = (SELECT id FROM call_request_types INNER JOIN registered_contracts ON call_requests.registered_contract_id = registered_contracts.id WHERE call_request_types.request_type = registered_contracts.contract_type);") + op.execute(f"INSERT INTO call_request_types (id, request_type) VALUES ('{str(uuid.uuid4())}', 'raw');") + op.execute(f"INSERT INTO call_request_types (id, request_type) VALUES ('{str(uuid.uuid4())}', 'dropper-v0.2.0');") + op.execute("UPDATE call_requests SET call_request_type_id = (SELECT call_request_types.id FROM call_request_types INNER JOIN registered_contracts ON call_requests.registered_contract_id = registered_contracts.id WHERE call_request_types.request_type = registered_contracts.contract_type);") op.alter_column("call_requests", "call_request_type_id", nullable=False) # Manual - End @@ -133,7 +135,7 @@ def downgrade(): # Copy type values # ... - op.drop_constraint(op.f('fk_call_requests_call_request_id_call_request_types'), 'call_requests', type_='foreignkey') + op.drop_constraint(op.f('fk_call_requests_call_request_type_id_call_request_types'), 'call_requests', type_='foreignkey') op.drop_column('call_requests', 'call_request_type_id') op.drop_index(op.f('ix_call_request_types_request_type'), table_name='call_request_types') From b0d8e1725496394d6fae79d826959d152b6dc7e8 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Aug 2023 09:59:43 +0000 Subject: [PATCH 15/50] Working downgrade --- ...f_call_request_types_and_metatx_holders.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py index 8ff6ba51..202d45cf 100644 --- a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py +++ b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py @@ -112,12 +112,13 @@ def downgrade(): # ### commands auto generated by Alembic - please adjust! ### # Blockchains - op.add_column('registered_contracts', sa.Column('blockchain', sa.VARCHAR(length=128), autoincrement=False, nullable=False)) + op.add_column('registered_contracts', sa.Column('blockchain', sa.VARCHAR(length=128), autoincrement=False, nullable=True)) op.create_index('ix_registered_contracts_blockchain', 'registered_contracts', ['blockchain'], unique=False) - # Manual - # Copy blockchain values - # ... + # Manual - Start + op.execute("UPDATE registered_contracts SET blockchain = (SELECT blockchains.name FROM blockchains WHERE blockchains.id = registered_contracts.blockchain_id);") + op.alter_column("registered_contracts", "blockchain", nullable=False) + # Manual - End op.drop_constraint(op.f('fk_registered_contracts_blockchain_id_blockchains'), 'registered_contracts', type_='foreignkey') op.drop_constraint(op.f('uq_registered_contracts_blockchain_id'), 'registered_contracts', type_='unique') @@ -128,12 +129,14 @@ def downgrade(): op.drop_table('blockchains') # Types - op.add_column('registered_contracts', sa.Column('contract_type', sa.VARCHAR(length=128), autoincrement=False, nullable=False)) + op.add_column('registered_contracts', sa.Column('contract_type', sa.VARCHAR(length=128), autoincrement=False, nullable=True)) op.create_index('ix_registered_contracts_contract_type', 'registered_contracts', ['contract_type'], unique=False) - # Manual - # Copy type values - # ... + # Manual - Start + # Hardcoded to set `dropper-v0.2.0` + op.execute("UPDATE registered_contracts SET contract_type = 'dropper-v0.2.0';") + op.alter_column("registered_contracts", "contract_type", nullable=False) + # Manual - End op.drop_constraint(op.f('fk_call_requests_call_request_type_id_call_request_types'), 'call_requests', type_='foreignkey') op.drop_column('call_requests', 'call_request_type_id') @@ -142,14 +145,17 @@ def downgrade(): op.drop_table('call_request_types') # Holders - op.add_column('registered_contracts', sa.Column('moonstream_user_id', sa.UUID(), autoincrement=False, nullable=False)) + op.add_column('registered_contracts', sa.Column('moonstream_user_id', sa.UUID(), autoincrement=False, nullable=True)) op.create_index('ix_registered_contracts_moonstream_user_id', 'registered_contracts', ['moonstream_user_id'], unique=False) - op.add_column('call_requests', sa.Column('moonstream_user_id', sa.UUID(), autoincrement=False, nullable=False)) + op.add_column('call_requests', sa.Column('moonstream_user_id', sa.UUID(), autoincrement=False, nullable=True)) op.create_index('ix_call_requests_moonstream_user_id', 'call_requests', ['moonstream_user_id'], unique=False) - # Manual - # Copy moonstream_user_ids - # ... + # Manual - Start + op.execute("UPDATE registered_contracts SET moonstream_user_id = metatx_holder_id;") + op.execute("UPDATE call_requests SET moonstream_user_id = metatx_holder_id;") + op.alter_column("registered_contracts", "moonstream_user_id", nullable=False) + op.alter_column("call_requests", "moonstream_user_id", nullable=False) + # Manual - End op.drop_constraint(op.f('fk_registered_contracts_metatx_holder_id_metatx_holders'), 'registered_contracts', type_='foreignkey') op.drop_column('registered_contracts', 'metatx_holder_id') From 6dfca4a49b6f465632bd3d4bfee86e066b416759 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Aug 2023 10:25:09 +0000 Subject: [PATCH 16/50] API list call request types --- ...f_call_request_types_and_metatx_holders.py | 5 ++- engineapi/engineapi/contracts_actions.py | 9 ++++- engineapi/engineapi/data.py | 13 +++++++ engineapi/engineapi/models.py | 1 + engineapi/engineapi/routes/metatx.py | 37 +++++++++++++------ 5 files changed, 51 insertions(+), 14 deletions(-) diff --git a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py index 202d45cf..a2f69860 100644 --- a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py +++ b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py @@ -52,6 +52,7 @@ def upgrade(): op.create_table('call_request_types', sa.Column('id', sa.UUID(), nullable=False), sa.Column('request_type', sa.VARCHAR(length=128), nullable=False), + sa.Column('description', sa.String(), nullable=True), sa.PrimaryKeyConstraint('id', name=op.f('pk_call_request_types')), sa.UniqueConstraint('id', name=op.f('uq_call_request_types_id')) ) @@ -62,8 +63,8 @@ def upgrade(): op.create_foreign_key(op.f('fk_call_requests_call_request_type_id_call_request_types'), 'call_requests', 'call_request_types', ['call_request_type_id'], ['id'], ondelete='CASCADE') # Manual - Start - op.execute(f"INSERT INTO call_request_types (id, request_type) VALUES ('{str(uuid.uuid4())}', 'raw');") - op.execute(f"INSERT INTO call_request_types (id, request_type) VALUES ('{str(uuid.uuid4())}', 'dropper-v0.2.0');") + op.execute(f"INSERT INTO call_request_types (id, request_type, description) VALUES ('{str(uuid.uuid4())}', 'raw', 'A generic smart contract. You can ask users to submit arbitrary calldata to this contract.');") + op.execute(f"INSERT INTO call_request_types (id, request_type, description) VALUES ('{str(uuid.uuid4())}', 'dropper-v0.2.0', 'A Dropper v0.2.0 contract. You can authorize users to submit claims against this contract.');") op.execute("UPDATE call_requests SET call_request_type_id = (SELECT call_request_types.id FROM call_request_types INNER JOIN registered_contracts ON call_requests.registered_contract_id = registered_contracts.id WHERE call_request_types.request_type = registered_contracts.contract_type);") op.alter_column("call_requests", "call_request_type_id", nullable=False) # Manual - End diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index 8e7ad875..cf4481a9 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -12,7 +12,7 @@ from web3 import Web3 from . import data, db from .data import ContractType -from .models import CallRequest, RegisteredContract +from .models import CallRequest, RegisteredContract, CallRequestType logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -334,6 +334,13 @@ def get_call_requests( ) +def list_call_request_types( + db_session: Session, +) -> List[CallRequestType]: + call_request_types = db_session.query(CallRequestType).all() + return call_request_types + + def list_call_requests( db_session: Session, contract_id: Optional[uuid.UUID], diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index b192bd3b..128ba0f5 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -208,6 +208,19 @@ class ContractType(Enum): dropper = "dropper-v0.2.0" +class CallRequestTypeResponse(BaseModel): + id: UUID + request_type: str + description: str + + class Config: + orm_mode = True + + +class CallRequestTypesResponse(BaseModel): + call_request_types: List[CallRequestTypeResponse] = Field(default_factory=list) + + class RegisterContractRequest(BaseModel): blockchain: str address: str diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index d98136cd..636c4e39 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -172,6 +172,7 @@ class CallRequestType(Base): # type: ignore unique=True, ) request_type = Column(VARCHAR(128), nullable=False, unique=True, index=True) + description = Column(String, nullable=True) call_requests = relationship( "CallRequest", diff --git a/engineapi/engineapi/routes/metatx.py b/engineapi/engineapi/routes/metatx.py index 18bc6a7b..9c75526a 100644 --- a/engineapi/engineapi/routes/metatx.py +++ b/engineapi/engineapi/routes/metatx.py @@ -38,6 +38,7 @@ whitelist_paths = { "/metatx/openapi.json": "GET", f"/metatx/{DOCS_TARGET_PATH}": "GET", "/metatx/contracts/types": "GET", + "/metatx/requests/types": "GET", "/metatx/requests": "GET", } @@ -62,17 +63,6 @@ app.add_middleware( ) -@app.get("/contracts/types", tags=["contracts"]) -async def contract_types() -> Dict[str, str]: - """ - Describes the contract_types that users can register contracts as against this API. - """ - return { - data.ContractType.raw.value: "A generic smart contract. You can ask users to submit arbitrary calldata to this contract.", - data.ContractType.dropper.value: "A Dropper contract. You can authorize users to submit claims against this contract.", - } - - @app.get("/contracts", tags=["contracts"], response_model=List[data.RegisteredContract]) async def list_registered_contracts( request: Request, @@ -219,6 +209,31 @@ async def delete_contract( return deleted_contract +# TODO(kompotkot): route `/contracts/types` deprecated +@app.get("/contracts/types", tags=["contracts"]) +@app.get( + "/requests/types", tags=["requests"], response_model=data.CallRequestTypesResponse +) +async def contract_types( + db_session: Session = Depends(db.yield_db_read_only_session), +) -> data.CallRequestTypesResponse: + """ + Describes the call_request_types that users can register call requests as against this API. + """ + try: + call_request_types = contracts_actions.list_call_request_types( + db_session=db_session, + ) + except Exception as e: + logger.error(repr(e)) + raise EngineHTTPException(status_code=500) + return data.CallRequestTypesResponse( + call_request_types=[ + call_request_type for call_request_type in call_request_types + ] + ) + + @app.get("/requests", tags=["requests"], response_model=List[data.CallRequest]) async def list_requests( contract_id: Optional[UUID] = Query(None), From 252da0cd36a38db02a574b503266076e2b7dda56 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Aug 2023 10:30:46 +0000 Subject: [PATCH 17/50] API route to get available blockchains for metatx --- engineapi/engineapi/contracts_actions.py | 9 +++++++- engineapi/engineapi/data.py | 14 +++++++++++++ engineapi/engineapi/routes/metatx.py | 26 +++++++++++++++++++++--- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index cf4481a9..42a48129 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -12,7 +12,7 @@ from web3 import Web3 from . import data, db from .data import ContractType -from .models import CallRequest, RegisteredContract, CallRequestType +from .models import Blockchain, CallRequest, CallRequestType, RegisteredContract logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -334,6 +334,13 @@ def get_call_requests( ) +def list_blockchains( + db_session: Session, +) -> List[Blockchain]: + blockchains = db_session.query(Blockchain).all() + return blockchains + + def list_call_request_types( db_session: Session, ) -> List[CallRequestType]: diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 128ba0f5..047e19d9 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -221,6 +221,20 @@ class CallRequestTypesResponse(BaseModel): call_request_types: List[CallRequestTypeResponse] = Field(default_factory=list) +class BlockchainResponse(BaseModel): + id: UUID + name: str + chain_id: int + testnet: bool + + class Config: + orm_mode = True + + +class BlockchainsResponse(BaseModel): + blockchains: List[BlockchainResponse] = Field(default_factory=list) + + class RegisterContractRequest(BaseModel): blockchain: str address: str diff --git a/engineapi/engineapi/routes/metatx.py b/engineapi/engineapi/routes/metatx.py index 9c75526a..37628628 100644 --- a/engineapi/engineapi/routes/metatx.py +++ b/engineapi/engineapi/routes/metatx.py @@ -9,12 +9,12 @@ import logging from typing import Dict, List, Optional from uuid import UUID -from fastapi import Body, Depends, FastAPI, Query, Request, Path +from fastapi import Body, Depends, FastAPI, Path, Query, Request from sqlalchemy.exc import NoResultFound from sqlalchemy.orm import Session from .. import contracts_actions, data, db -from ..middleware import BroodAuthMiddleware, EngineHTTPException, BugoutCORSMiddleware +from ..middleware import BroodAuthMiddleware, BugoutCORSMiddleware, EngineHTTPException from ..settings import DOCS_TARGET_PATH from ..version import VERSION @@ -37,6 +37,7 @@ tags_metadata = [ whitelist_paths = { "/metatx/openapi.json": "GET", f"/metatx/{DOCS_TARGET_PATH}": "GET", + "/metatx/blockchains": "GET", "/metatx/contracts/types": "GET", "/metatx/requests/types": "GET", "/metatx/requests": "GET", @@ -63,6 +64,25 @@ app.add_middleware( ) +@app.get("/blockchains", tags=["blockchains"], response_model=data.BlockchainsResponse) +async def blockchains_route( + db_session: Session = Depends(db.yield_db_read_only_session), +) -> data.BlockchainsResponse: + """ + Returns supported list of blockchains. + """ + try: + blockchains = contracts_actions.list_blockchains( + db_session=db_session, + ) + except Exception as e: + logger.error(repr(e)) + raise EngineHTTPException(status_code=500) + return data.BlockchainsResponse( + blockchains=[blockchain for blockchain in blockchains] + ) + + @app.get("/contracts", tags=["contracts"], response_model=List[data.RegisteredContract]) async def list_registered_contracts( request: Request, @@ -214,7 +234,7 @@ async def delete_contract( @app.get( "/requests/types", tags=["requests"], response_model=data.CallRequestTypesResponse ) -async def contract_types( +async def call_request_types_route( db_session: Session = Depends(db.yield_db_read_only_session), ) -> data.CallRequestTypesResponse: """ From 47db2e534c270d7156195fbbb1eb523b64410159 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Aug 2023 10:42:16 +0000 Subject: [PATCH 18/50] Updated API route for list contracts --- engineapi/engineapi/contracts_actions.py | 22 ++++++------- engineapi/engineapi/data.py | 28 ++++++++++++++-- engineapi/engineapi/routes/metatx.py | 42 +++++++++++++++++------- 3 files changed, 65 insertions(+), 27 deletions(-) diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index 42a48129..feb02220 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -3,7 +3,7 @@ import json import logging import uuid from datetime import timedelta -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from sqlalchemy import func, text from sqlalchemy.exc import IntegrityError, NoResultFound @@ -165,37 +165,35 @@ def get_registered_contract( def lookup_registered_contracts( db_session: Session, - moonstream_user_id: uuid.UUID, + metatx_holder_id: uuid.UUID, blockchain: Optional[str] = None, address: Optional[str] = None, - contract_type: Optional[ContractType] = None, limit: int = 10, offset: Optional[int] = None, -) -> List[RegisteredContract]: +) -> List[Tuple[RegisteredContract, Blockchain]]: """ Lookup a registered contract """ - query = db_session.query(RegisteredContract).filter( - RegisteredContract.moonstream_user_id == moonstream_user_id + query = ( + db_session.query(RegisteredContract, Blockchain) + .join(Blockchain, Blockchain.id == RegisteredContract.blockchain_id) + .filter(RegisteredContract.metatx_holder_id == metatx_holder_id) ) if blockchain is not None: - query = query.filter(RegisteredContract.blockchain == blockchain) + query = query.filter(Blockchain.name == blockchain) if address is not None: query = query.filter( RegisteredContract.address == Web3.toChecksumAddress(address) ) - if contract_type is not None: - query = query.filter(RegisteredContract.contract_type == contract_type.value) - if offset is not None: query = query.offset(offset) - query = query.limit(limit) + registered_contracts_with_blockchain = query.limit(limit).all() - return query.all() + return registered_contracts_with_blockchain def delete_registered_contract( diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 047e19d9..f519e218 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -255,15 +255,37 @@ class RegisteredContract(BaseModel): id: UUID blockchain: str address: str - contract_type: str - moonstream_user_id: UUID + metatx_holder_id: UUID title: Optional[str] = None description: Optional[str] = None image_uri: Optional[str] = None created_at: datetime updated_at: datetime - @validator("id", "moonstream_user_id") + @validator("id", "metatx_holder_id") + def validate_uuids(cls, v): + return str(v) + + @validator("created_at", "updated_at") + def validate_datetimes(cls, v): + return v.isoformat() + + class Config: + orm_mode = True + + +class RegisteredContractResponse(BaseModel): + id: UUID + blockchain: Optional[str] = None + address: str + metatx_holder_id: UUID + title: Optional[str] = None + description: Optional[str] = None + image_uri: Optional[str] = None + created_at: datetime + updated_at: datetime + + @validator("id", "metatx_holder_id") def validate_uuids(cls, v): return str(v) diff --git a/engineapi/engineapi/routes/metatx.py b/engineapi/engineapi/routes/metatx.py index 37628628..8701a539 100644 --- a/engineapi/engineapi/routes/metatx.py +++ b/engineapi/engineapi/routes/metatx.py @@ -83,33 +83,51 @@ async def blockchains_route( ) -@app.get("/contracts", tags=["contracts"], response_model=List[data.RegisteredContract]) +@app.get( + "/contracts", + tags=["contracts"], + response_model=List[data.RegisteredContractResponse], +) async def list_registered_contracts( request: Request, blockchain: Optional[str] = Query(None), address: Optional[str] = Query(None), - contract_type: Optional[data.ContractType] = Query(None), limit: int = Query(10), offset: Optional[int] = Query(None), db_session: Session = Depends(db.yield_db_read_only_session), -) -> List[data.RegisteredContract]: +) -> List[data.RegisteredContractResponse]: """ Users can use this endpoint to look up the contracts they have registered against this API. """ try: - contracts = contracts_actions.lookup_registered_contracts( - db_session=db_session, - moonstream_user_id=request.state.user.id, - blockchain=blockchain, - address=address, - contract_type=contract_type, - limit=limit, - offset=offset, + registered_contracts_with_blockchain = ( + contracts_actions.lookup_registered_contracts( + db_session=db_session, + metatx_holder_id=request.state.user.id, + blockchain=blockchain, + address=address, + limit=limit, + offset=offset, + ) ) except Exception as err: logger.error(repr(err)) raise EngineHTTPException(status_code=500) - return [contract for contract in contracts] + + return [ + data.RegisteredContractResponse( + id=rc_with_b[0].id, + blockchain=rc_with_b[1].name, + address=rc_with_b[0].address, + metatx_holder_id=rc_with_b[0].metatx_holder_id, + title=rc_with_b[0].title, + description=rc_with_b[0].description, + image_uri=rc_with_b[0].image_uri, + created_at=rc_with_b[0].created_at, + updated_at=rc_with_b[0].updated_at, + ) + for rc_with_b in registered_contracts_with_blockchain + ] @app.get( From 1589c8c65ddba531ea26720a3aada4f91f918b5b Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Aug 2023 12:36:13 +0000 Subject: [PATCH 19/50] Fixed metatx contract routes to work with new db model --- engineapi/engineapi/contracts_actions.py | 111 +++++++++++++-------- engineapi/engineapi/data.py | 24 ----- engineapi/engineapi/routes/metatx.py | 118 ++++++++++++++++------- 3 files changed, 157 insertions(+), 96 deletions(-) diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index feb02220..6427ff74 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -6,13 +6,21 @@ from datetime import timedelta from typing import Any, Dict, List, Optional, Tuple from sqlalchemy import func, text +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.engine import Row from sqlalchemy.exc import IntegrityError, NoResultFound from sqlalchemy.orm import Session from web3 import Web3 from . import data, db from .data import ContractType -from .models import Blockchain, CallRequest, CallRequestType, RegisteredContract +from .models import ( + Blockchain, + CallRequest, + CallRequestType, + MetatxHolder, + RegisteredContract, +) logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -30,6 +38,12 @@ class InvalidAddressFormat(Exception): """ +class UnsupportedBlockchain(Exception): + """ + Raised when unsupported blockchain specified. + """ + + class ContractAlreadyRegistered(Exception): pass @@ -72,23 +86,33 @@ def validate_method_and_params( def register_contract( db_session: Session, - blockchain: str, + blockchain_name: str, address: str, - contract_type: ContractType, - moonstream_user_id: uuid.UUID, + metatx_holder_id: uuid.UUID, title: Optional[str], description: Optional[str], image_uri: Optional[str], -) -> data.RegisteredContract: +) -> Tuple[RegisteredContract, Blockchain]: """ Register a contract against the Engine instance """ try: + blockchain = ( + db_session.query(Blockchain) + .filter(Blockchain.name == blockchain_name) + .one_or_none() + ) + if blockchain is None: + raise UnsupportedBlockchain("Unsupported blockchain specified") + + metatx_holder_stmt = insert(MetatxHolder.__table__).values(id=metatx_holder_id) + metatx_holder_stmt_do_nothing_stmt = metatx_holder_stmt.on_conflict_do_nothing() + db_session.execute(metatx_holder_stmt_do_nothing_stmt) + contract = RegisteredContract( - blockchain=blockchain, + blockchain_id=blockchain.id, address=Web3.toChecksumAddress(address), - contract_type=contract_type.value, - moonstream_user_id=moonstream_user_id, + metatx_holder_id=metatx_holder_id, title=title, description=description, image_uri=image_uri, @@ -103,38 +127,42 @@ def register_contract( logger.error(repr(err)) raise - return contract + return (contract, blockchain) def update_registered_contract( db_session: Session, - moonstream_user_id: uuid.UUID, + metatx_holder_id: uuid.UUID, contract_id: uuid.UUID, title: Optional[str] = None, description: Optional[str] = None, image_uri: Optional[str] = None, ignore_nulls: bool = True, -) -> data.RegisteredContract: +) -> Tuple[RegisteredContract, Blockchain]: """ - Update the registered contract with the given contract ID provided that the user with moonstream_user_id + Update the registered contract with the given contract ID provided that the user with metatx_holder_id has access to it. """ - query = db_session.query(RegisteredContract).filter( - RegisteredContract.id == contract_id, - RegisteredContract.moonstream_user_id == moonstream_user_id, + contract_with_blockchain = ( + db_session.query(RegisteredContract, Blockchain) + .join(Blockchain, Blockchain.id == RegisteredContract.blockchain_id) + .filter( + RegisteredContract.id == contract_id, + RegisteredContract.metatx_holder_id == metatx_holder_id, + ) + .one() ) - - contract = query.one() + registered_contract, blockchain = contract_with_blockchain if not (title is None and ignore_nulls): - contract.title = title + registered_contract.title = title if not (description is None and ignore_nulls): - contract.description = description + registered_contract.description = description if not (image_uri is None and ignore_nulls): - contract.image_uri = image_uri + registered_contract.image_uri = image_uri try: - db_session.add(contract) + db_session.add(registered_contract) db_session.commit() except Exception as err: logger.error( @@ -143,24 +171,27 @@ def update_registered_contract( db_session.rollback() raise - return contract + return (registered_contract, blockchain) def get_registered_contract( db_session: Session, - moonstream_user_id: uuid.UUID, + metatx_holder_id: uuid.UUID, contract_id: uuid.UUID, -) -> RegisteredContract: +) -> Tuple[RegisteredContract, Blockchain]: """ Get registered contract by ID. """ - contract = ( - db_session.query(RegisteredContract) - .filter(RegisteredContract.moonstream_user_id == moonstream_user_id) + contract_with_blockchain = ( + db_session.query(RegisteredContract, Blockchain) + .join(Blockchain, Blockchain.id == RegisteredContract.blockchain_id) + .filter(RegisteredContract.metatx_holder_id == metatx_holder_id) .filter(RegisteredContract.id == contract_id) .one() ) - return contract + registered_contract, blockchain = contract_with_blockchain + + return (registered_contract, blockchain) def lookup_registered_contracts( @@ -170,7 +201,7 @@ def lookup_registered_contracts( address: Optional[str] = None, limit: int = 10, offset: Optional[int] = None, -) -> List[Tuple[RegisteredContract, Blockchain]]: +) -> List[Row[Tuple[RegisteredContract, Blockchain]]]: """ Lookup a registered contract """ @@ -191,35 +222,39 @@ def lookup_registered_contracts( if offset is not None: query = query.offset(offset) - registered_contracts_with_blockchain = query.limit(limit).all() + contracts_with_blockchain = query.limit(limit).all() - return registered_contracts_with_blockchain + return contracts_with_blockchain def delete_registered_contract( db_session: Session, - moonstream_user_id: uuid.UUID, + metatx_holder_id: uuid.UUID, registered_contract_id: uuid.UUID, -) -> RegisteredContract: +) -> Tuple[RegisteredContract, Blockchain]: """ Delete a registered contract """ try: - registered_contract = ( - db_session.query(RegisteredContract) - .filter(RegisteredContract.moonstream_user_id == moonstream_user_id) + contract_with_blockchain = ( + db_session.query(RegisteredContract, Blockchain) + .join(Blockchain, Blockchain.id == RegisteredContract.blockchain_id) + .filter(RegisteredContract.metatx_holder_id == metatx_holder_id) .filter(RegisteredContract.id == registered_contract_id) .one() ) + contract = contract_with_blockchain[0] - db_session.delete(registered_contract) + db_session.delete(contract) db_session.commit() except Exception as err: db_session.rollback() logger.error(repr(err)) raise - return registered_contract + registered_contract, blockchain = contract_with_blockchain + + return (registered_contract, blockchain) def request_calls( diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index f519e218..0a646ac1 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -238,7 +238,6 @@ class BlockchainsResponse(BaseModel): class RegisterContractRequest(BaseModel): blockchain: str address: str - contract_type: ContractType title: Optional[str] = None description: Optional[str] = None image_uri: Optional[str] = None @@ -251,29 +250,6 @@ class UpdateContractRequest(BaseModel): ignore_nulls: bool = True -class RegisteredContract(BaseModel): - id: UUID - blockchain: str - address: str - metatx_holder_id: UUID - title: Optional[str] = None - description: Optional[str] = None - image_uri: Optional[str] = None - created_at: datetime - updated_at: datetime - - @validator("id", "metatx_holder_id") - def validate_uuids(cls, v): - return str(v) - - @validator("created_at", "updated_at") - def validate_datetimes(cls, v): - return v.isoformat() - - class Config: - orm_mode = True - - class RegisteredContractResponse(BaseModel): id: UUID blockchain: Optional[str] = None diff --git a/engineapi/engineapi/routes/metatx.py b/engineapi/engineapi/routes/metatx.py index 8701a539..d8f7577d 100644 --- a/engineapi/engineapi/routes/metatx.py +++ b/engineapi/engineapi/routes/metatx.py @@ -88,7 +88,7 @@ async def blockchains_route( tags=["contracts"], response_model=List[data.RegisteredContractResponse], ) -async def list_registered_contracts( +async def list_registered_contracts_route( request: Request, blockchain: Optional[str] = Query(None), address: Optional[str] = Query(None), @@ -133,20 +133,20 @@ async def list_registered_contracts( @app.get( "/contracts/{contract_id}", tags=["contracts"], - response_model=data.RegisteredContract, + response_model=data.RegisteredContractResponse, ) -async def get_registered_contract( +async def get_registered_contract_route( request: Request, contract_id: UUID = Path(...), db_session: Session = Depends(db.yield_db_read_only_session), -) -> List[data.RegisteredContract]: +) -> List[data.RegisteredContractResponse]: """ Get the contract by ID. """ try: - contract = contracts_actions.get_registered_contract( + contract_with_blockchain = contracts_actions.get_registered_contract( db_session=db_session, - moonstream_user_id=request.state.user.id, + metatx_holder_id=request.state.user.id, contract_id=contract_id, ) except NoResultFound: @@ -157,57 +157,87 @@ async def get_registered_contract( except Exception as err: logger.error(repr(err)) raise EngineHTTPException(status_code=500) - return contract + + return data.RegisteredContractResponse( + id=contract_with_blockchain[0].id, + blockchain=contract_with_blockchain[1].name, + address=contract_with_blockchain[0].address, + metatx_holder_id=contract_with_blockchain[0].metatx_holder_id, + title=contract_with_blockchain[0].title, + description=contract_with_blockchain[0].description, + image_uri=contract_with_blockchain[0].image_uri, + created_at=contract_with_blockchain[0].created_at, + updated_at=contract_with_blockchain[0].updated_at, + ) -@app.post("/contracts", tags=["contracts"], response_model=data.RegisteredContract) -async def register_contract( +@app.post( + "/contracts", tags=["contracts"], response_model=data.RegisteredContractResponse +) +async def register_contract_route( request: Request, contract: data.RegisterContractRequest = Body(...), db_session: Session = Depends(db.yield_db_session), -) -> data.RegisteredContract: +) -> data.RegisteredContractResponse: """ Allows users to register contracts. """ try: - registered_contract = contracts_actions.register_contract( + contract_with_blockchain = contracts_actions.register_contract( db_session=db_session, - moonstream_user_id=request.state.user.id, - blockchain=contract.blockchain, + metatx_holder_id=request.state.user.id, + blockchain_name=contract.blockchain, address=contract.address, - contract_type=contract.contract_type, title=contract.title, description=contract.description, image_uri=contract.image_uri, ) + except contracts_actions.UnsupportedBlockchain: + raise EngineHTTPException( + status_code=400, detail="Unsupported blockchain specified" + ) except contracts_actions.ContractAlreadyRegistered: raise EngineHTTPException( status_code=409, detail="Contract already registered", ) - return registered_contract + except Exception as err: + logger.error(repr(err)) + raise EngineHTTPException(status_code=500) + + return data.RegisteredContractResponse( + id=contract_with_blockchain[0].id, + blockchain=contract_with_blockchain[1].name, + address=contract_with_blockchain[0].address, + metatx_holder_id=contract_with_blockchain[0].metatx_holder_id, + title=contract_with_blockchain[0].title, + description=contract_with_blockchain[0].description, + image_uri=contract_with_blockchain[0].image_uri, + created_at=contract_with_blockchain[0].created_at, + updated_at=contract_with_blockchain[0].updated_at, + ) @app.put( "/contracts/{contract_id}", tags=["contracts"], - response_model=data.RegisteredContract, + response_model=data.RegisteredContractResponse, ) -async def update_contract( +async def update_contract_route( request: Request, contract_id: UUID = Path(...), update_info: data.UpdateContractRequest = Body(...), db_session: Session = Depends(db.yield_db_session), -) -> data.RegisteredContract: +) -> data.RegisteredContractResponse: try: - contract = contracts_actions.update_registered_contract( - db_session, - request.state.user.id, - contract_id, - update_info.title, - update_info.description, - update_info.image_uri, - update_info.ignore_nulls, + contract_with_blockchain = contracts_actions.update_registered_contract( + db_session=db_session, + metatx_holder_id=request.state.user.id, + contract_id=contract_id, + title=update_info.title, + description=update_info.description, + image_uri=update_info.image_uri, + ignore_nulls=update_info.ignore_nulls, ) except NoResultFound: raise EngineHTTPException( @@ -218,33 +248,53 @@ async def update_contract( logger.error(repr(err)) raise EngineHTTPException(status_code=500) - return contract + return data.RegisteredContractResponse( + id=contract_with_blockchain[0].id, + blockchain=contract_with_blockchain[1].name, + address=contract_with_blockchain[0].address, + metatx_holder_id=contract_with_blockchain[0].metatx_holder_id, + title=contract_with_blockchain[0].title, + description=contract_with_blockchain[0].description, + image_uri=contract_with_blockchain[0].image_uri, + created_at=contract_with_blockchain[0].created_at, + updated_at=contract_with_blockchain[0].updated_at, + ) @app.delete( "/contracts/{contract_id}", tags=["contracts"], - response_model=data.RegisteredContract, + response_model=data.RegisteredContractResponse, ) -async def delete_contract( +async def delete_contract_route( request: Request, - contract_id: UUID, + contract_id: UUID = Path(...), db_session: Session = Depends(db.yield_db_session), -) -> data.RegisteredContract: +) -> data.RegisteredContractResponse: """ Allows users to delete contracts that they have registered. """ try: - deleted_contract = contracts_actions.delete_registered_contract( + deleted_contract_with_blockchain = contracts_actions.delete_registered_contract( db_session=db_session, - moonstream_user_id=request.state.user.id, + metatx_holder_id=request.state.user.id, registered_contract_id=contract_id, ) except Exception as err: logger.error(repr(err)) raise EngineHTTPException(status_code=500) - return deleted_contract + return data.RegisteredContractResponse( + id=deleted_contract_with_blockchain[0].id, + blockchain=deleted_contract_with_blockchain[1].name, + address=deleted_contract_with_blockchain[0].address, + metatx_holder_id=deleted_contract_with_blockchain[0].metatx_holder_id, + title=deleted_contract_with_blockchain[0].title, + description=deleted_contract_with_blockchain[0].description, + image_uri=deleted_contract_with_blockchain[0].image_uri, + created_at=deleted_contract_with_blockchain[0].created_at, + updated_at=deleted_contract_with_blockchain[0].updated_at, + ) # TODO(kompotkot): route `/contracts/types` deprecated From 927eb85c94437210ed44b68ffe98d3489b84294e Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 3 Aug 2023 14:12:06 +0000 Subject: [PATCH 20/50] Basic call requests endpoints to work with updated db --- engineapi/engineapi/contracts_actions.py | 76 ++++++++++++++----- engineapi/engineapi/data.py | 11 +-- engineapi/engineapi/routes/metatx.py | 96 +++++++----------------- 3 files changed, 90 insertions(+), 93 deletions(-) diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index 6427ff74..5c7a8476 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -48,6 +48,40 @@ class ContractAlreadyRegistered(Exception): pass +def parse_registered_contract_response( + obj: Tuple[RegisteredContract, Blockchain] +) -> data.RegisteredContractResponse: + return data.RegisteredContractResponse( + id=obj[0].id, + blockchain=obj[1].name, + address=obj[0].address, + metatx_holder_id=obj[0].metatx_holder_id, + title=obj[0].title, + description=obj[0].description, + image_uri=obj[0].image_uri, + created_at=obj[0].created_at, + updated_at=obj[0].updated_at, + ) + + +def parse_call_request_response( + obj: Tuple[CallRequest, RegisteredContract, CallRequestType, CallRequestType] +) -> data.CallRequestResponse: + return data.CallRequestResponse( + id=obj[0].id, + contract_id=obj[0].registered_contract_id, + contract_address=obj[1].address, + metatx_holder_id=obj[0].metatx_holder_id, + call_request_type=obj[2].request_type, + caller=obj[0].caller, + method=obj[0].method, + parameters=obj[0].parameters, + expires_at=obj[0].expires_at, + created_at=obj[0].created_at, + updated_at=obj[0].updated_at, + ) + + def validate_method_and_params( contract_type: ContractType, method: str, parameters: Dict[str, Any] ) -> None: @@ -259,7 +293,7 @@ def delete_registered_contract( def request_calls( db_session: Session, - moonstream_user_id: uuid.UUID, + metatx_holder_id: uuid.UUID, registered_contract_id: Optional[uuid.UUID], contract_address: Optional[str], call_specs: List[data.CallSpecification], @@ -283,7 +317,7 @@ def request_calls( # Check that the moonstream_user_id matches a RegisteredContract with the given id or address query = db_session.query(RegisteredContract).filter( - RegisteredContract.moonstream_user_id == moonstream_user_id + RegisteredContract.metatx_holder_id == metatx_holder_id ) if registered_contract_id is not None: @@ -297,7 +331,7 @@ def request_calls( try: registered_contract = query.one() except NoResultFound: - raise ValueError("Invalid registered_contract_id or moonstream_user_id") + raise ValueError("Invalid registered_contract_id or metatx_holder_id") # Normalize the caller argument using Web3.toChecksumAddress contract_type = ContractType(registered_contract.contract_type) @@ -343,16 +377,20 @@ def request_calls( def get_call_requests( db_session: Session, request_id: uuid.UUID, -) -> data.CallRequest: +) -> Tuple[CallRequest, RegisteredContract, CallRequestType, CallRequestType]: """ Get call request by ID. """ results = ( - db_session.query(CallRequest, RegisteredContract) + db_session.query(CallRequest, RegisteredContract, CallRequestType) .join( RegisteredContract, CallRequest.registered_contract_id == RegisteredContract.id, ) + .join( + CallRequestType, + CallRequest.call_request_type_id == CallRequestType.id, + ) .filter(CallRequest.id == request_id) .all() ) @@ -360,11 +398,12 @@ def get_call_requests( raise CallRequestNotFound("Call request with given ID not found") elif len(results) != 1: raise Exception( - f"Incorrect number of results found for moonstream_user_id {moonstream_user_id} and request_id {request_id}" + f"Incorrect number of results found for request_id {request_id}" ) - return data.CallRequest( - contract_address=results[0][1].address, **results[0][0].__dict__ - ) + + call_request, registered_contract, call_request_type = results[0] + + return (call_request, registered_contract, call_request_type) def list_blockchains( @@ -389,7 +428,7 @@ def list_call_requests( limit: int = 10, offset: Optional[int] = None, show_expired: bool = False, -) -> List[data.CallRequest]: +) -> List[Row[Tuple[CallRequest, RegisteredContract, CallRequestType]]]: """ List call requests for the given moonstream_user_id """ @@ -403,11 +442,15 @@ def list_call_requests( # If show_expired is False, filter out expired requests using current time on database server query = ( - db_session.query(CallRequest, RegisteredContract) + db_session.query(CallRequest, RegisteredContract, CallRequestType) .join( RegisteredContract, CallRequest.registered_contract_id == RegisteredContract.id, ) + .join( + CallRequestType, + CallRequest.call_request_type_id == CallRequestType.id, + ) .filter(CallRequest.caller == Web3.toChecksumAddress(caller)) ) @@ -429,12 +472,7 @@ def list_call_requests( query = query.limit(limit) results = query.all() - return [ - data.CallRequest( - contract_address=registered_contract.address, **call_request.__dict__ - ) - for call_request, registered_contract in results - ] + return results # TODO(zomglings): What should the delete functionality for call requests look like? @@ -449,7 +487,7 @@ def list_call_requests( def delete_requests( db_session: Session, - moonstream_user_id: uuid.UUID, + metatx_holder_id: uuid.UUID, request_ids: List[uuid.UUID] = [], ) -> int: """ @@ -458,7 +496,7 @@ def delete_requests( try: requests_to_delete_query = ( db_session.query(CallRequest) - .filter(CallRequest.moonstream_user_id == moonstream_user_id) + .filter(CallRequest.metatx_holder_id == metatx_holder_id) .filter(CallRequest.id.in_(request_ids)) ) requests_to_delete_num: int = requests_to_delete_query.delete( diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 0a646ac1..65be0006 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -299,22 +299,23 @@ class CreateCallRequestsAPIRequest(BaseModel): return values -class CallRequest(BaseModel): +class CallRequestResponse(BaseModel): id: UUID - contract_id: UUID = Field(alias="registered_contract_id") + contract_id: UUID contract_address: Optional[str] = None - moonstream_user_id: UUID + metatx_holder_id: UUID + call_request_type: Optional[str] = None caller: str method: str parameters: Dict[str, Any] - expires_at: Optional[datetime] + expires_at: Optional[datetime] = None created_at: datetime updated_at: datetime class Config: orm_mode = True - @validator("id", "contract_id", "moonstream_user_id") + @validator("id", "contract_id", "metatx_holder_id") def validate_uuids(cls, v): return str(v) diff --git a/engineapi/engineapi/routes/metatx.py b/engineapi/engineapi/routes/metatx.py index d8f7577d..35841266 100644 --- a/engineapi/engineapi/routes/metatx.py +++ b/engineapi/engineapi/routes/metatx.py @@ -115,18 +115,8 @@ async def list_registered_contracts_route( raise EngineHTTPException(status_code=500) return [ - data.RegisteredContractResponse( - id=rc_with_b[0].id, - blockchain=rc_with_b[1].name, - address=rc_with_b[0].address, - metatx_holder_id=rc_with_b[0].metatx_holder_id, - title=rc_with_b[0].title, - description=rc_with_b[0].description, - image_uri=rc_with_b[0].image_uri, - created_at=rc_with_b[0].created_at, - updated_at=rc_with_b[0].updated_at, - ) - for rc_with_b in registered_contracts_with_blockchain + contracts_actions.parse_registered_contract_response(rc) + for rc in registered_contracts_with_blockchain ] @@ -158,16 +148,8 @@ async def get_registered_contract_route( logger.error(repr(err)) raise EngineHTTPException(status_code=500) - return data.RegisteredContractResponse( - id=contract_with_blockchain[0].id, - blockchain=contract_with_blockchain[1].name, - address=contract_with_blockchain[0].address, - metatx_holder_id=contract_with_blockchain[0].metatx_holder_id, - title=contract_with_blockchain[0].title, - description=contract_with_blockchain[0].description, - image_uri=contract_with_blockchain[0].image_uri, - created_at=contract_with_blockchain[0].created_at, - updated_at=contract_with_blockchain[0].updated_at, + return contracts_actions.parse_registered_contract_response( + contract_with_blockchain ) @@ -205,16 +187,8 @@ async def register_contract_route( logger.error(repr(err)) raise EngineHTTPException(status_code=500) - return data.RegisteredContractResponse( - id=contract_with_blockchain[0].id, - blockchain=contract_with_blockchain[1].name, - address=contract_with_blockchain[0].address, - metatx_holder_id=contract_with_blockchain[0].metatx_holder_id, - title=contract_with_blockchain[0].title, - description=contract_with_blockchain[0].description, - image_uri=contract_with_blockchain[0].image_uri, - created_at=contract_with_blockchain[0].created_at, - updated_at=contract_with_blockchain[0].updated_at, + return contracts_actions.parse_registered_contract_response( + contract_with_blockchain ) @@ -248,16 +222,8 @@ async def update_contract_route( logger.error(repr(err)) raise EngineHTTPException(status_code=500) - return data.RegisteredContractResponse( - id=contract_with_blockchain[0].id, - blockchain=contract_with_blockchain[1].name, - address=contract_with_blockchain[0].address, - metatx_holder_id=contract_with_blockchain[0].metatx_holder_id, - title=contract_with_blockchain[0].title, - description=contract_with_blockchain[0].description, - image_uri=contract_with_blockchain[0].image_uri, - created_at=contract_with_blockchain[0].created_at, - updated_at=contract_with_blockchain[0].updated_at, + return contracts_actions.parse_registered_contract_response( + contract_with_blockchain ) @@ -284,27 +250,21 @@ async def delete_contract_route( logger.error(repr(err)) raise EngineHTTPException(status_code=500) - return data.RegisteredContractResponse( - id=deleted_contract_with_blockchain[0].id, - blockchain=deleted_contract_with_blockchain[1].name, - address=deleted_contract_with_blockchain[0].address, - metatx_holder_id=deleted_contract_with_blockchain[0].metatx_holder_id, - title=deleted_contract_with_blockchain[0].title, - description=deleted_contract_with_blockchain[0].description, - image_uri=deleted_contract_with_blockchain[0].image_uri, - created_at=deleted_contract_with_blockchain[0].created_at, - updated_at=deleted_contract_with_blockchain[0].updated_at, + return contracts_actions.parse_registered_contract_response( + deleted_contract_with_blockchain ) # TODO(kompotkot): route `/contracts/types` deprecated @app.get("/contracts/types", tags=["contracts"]) @app.get( - "/requests/types", tags=["requests"], response_model=data.CallRequestTypesResponse + "/requests/types", + tags=["requests"], + response_model=List[data.CallRequestTypeResponse], ) async def call_request_types_route( db_session: Session = Depends(db.yield_db_read_only_session), -) -> data.CallRequestTypesResponse: +) -> List[data.CallRequestTypeResponse]: """ Describes the call_request_types that users can register call requests as against this API. """ @@ -315,15 +275,11 @@ async def call_request_types_route( except Exception as e: logger.error(repr(e)) raise EngineHTTPException(status_code=500) - return data.CallRequestTypesResponse( - call_request_types=[ - call_request_type for call_request_type in call_request_types - ] - ) + return call_request_types -@app.get("/requests", tags=["requests"], response_model=List[data.CallRequest]) -async def list_requests( +@app.get("/requests", tags=["requests"], response_model=List[data.CallRequestResponse]) +async def list_requests_route( contract_id: Optional[UUID] = Query(None), contract_address: Optional[str] = Query(None), caller: str = Query(...), @@ -331,7 +287,7 @@ async def list_requests( offset: Optional[int] = Query(None), show_expired: Optional[bool] = Query(False), db_session: Session = Depends(db.yield_db_read_only_session), -) -> List[data.CallRequest]: +) -> List[data.CallRequestResponse]: """ Allows API user to see all unexpired call requests for a given caller against a given contract. @@ -354,21 +310,23 @@ async def list_requests( logger.error(repr(e)) raise EngineHTTPException(status_code=500) - return requests + return [contracts_actions.parse_call_request_response(r) for r in requests] -@app.get("/requests/{request_id}", tags=["requests"], response_model=data.CallRequest) +@app.get( + "/requests/{request_id}", tags=["requests"], response_model=data.CallRequestResponse +) async def get_request( request_id: UUID = Path(...), db_session: Session = Depends(db.yield_db_read_only_session), -) -> List[data.CallRequest]: +) -> List[data.CallRequestResponse]: """ Allows API user to see call request. At least one of `contract_id` or `contract_address` must be provided as query parameters. """ try: - result = contracts_actions.get_call_requests( + request = contracts_actions.get_call_requests( db_session=db_session, request_id=request_id, ) @@ -381,7 +339,7 @@ async def get_request( logger.error(repr(e)) raise EngineHTTPException(status_code=500) - return result + return contracts_actions.parse_call_request_response(request) @app.post("/requests", tags=["requests"], response_model=int) @@ -398,7 +356,7 @@ async def create_requests( try: num_requests = contracts_actions.request_calls( db_session=db_session, - moonstream_user_id=request.state.user.id, + metatx_holder_id=request.state.user.id, registered_contract_id=data.contract_id, contract_address=data.contract_address, call_specs=data.specifications, @@ -428,7 +386,7 @@ async def delete_requests( try: deleted_requests = contracts_actions.delete_requests( db_session=db_session, - moonstream_user_id=request.state.user.id, + metatx_holder_id=request.state.user.id, request_ids=request_ids, ) except Exception as err: From ef57400c521199a0c529d64008e725868fc95066 Mon Sep 17 00:00:00 2001 From: Sergei Sumarokov <7047457+kompotkot@users.noreply.github.com> Date: Fri, 4 Aug 2023 12:53:35 -0700 Subject: [PATCH 21/50] Update leaderboard.py docs whitelisted --- engineapi/engineapi/routes/leaderboard.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 648015fc..19f12099 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -30,6 +30,8 @@ tags_metadata = [ leaderboad_whitelist = { + f"/leaderboard/{DOCS_TARGET_PATH}": "GET", + "/leaderboard/openapi.json": "GET", "/leaderboard/info": "GET", "/leaderboard/quartiles": "GET", "/leaderboard/count/addresses": "GET", From 0403b1ba0cbe726f4a9a7448e755cccb60a3b21b Mon Sep 17 00:00:00 2001 From: Andrey Date: Sun, 6 Aug 2023 16:55:56 +0300 Subject: [PATCH 22/50] Add initial leaderboard manage. --- engineapi/engineapi/actions.py | 213 ++++++++++++++++++---- engineapi/engineapi/data.py | 43 +++++ engineapi/engineapi/routes/leaderboard.py | 188 ++++++++++++++++++- 3 files changed, 412 insertions(+), 32 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 168d6ad6..e981a565 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -64,6 +64,18 @@ class LeaderboardDeleteScoresError(Exception): pass +class LeaderboardCreateError(Exception): + pass + + +class LeaderboardUpdateError(Exception): + pass + + +class LeaderboardDeleteError(Exception): + pass + + BATCH_SIGNATURE_PAGE_SIZE = 500 logger = logging.getLogger(__name__) @@ -208,7 +220,7 @@ def delete_claim(db_session: Session, dropper_claim_id): """ claim = ( - db_session.query(DropperClaim).filter(DropperClaim.id == dropper_claim_id).one() + db_session.query(DropperClaim).filter(DropperClaim.id == dropper_claim_id).one() # type: ignore ) db_session.delete(claim) @@ -260,7 +272,7 @@ def activate_drop(db_session: Session, dropper_claim_id: uuid.UUID): """ claim = ( - db_session.query(DropperClaim).filter(DropperClaim.id == dropper_claim_id).one() + db_session.query(DropperClaim).filter(DropperClaim.id == dropper_claim_id).one() # type: ignore ) claim.active = True @@ -275,7 +287,7 @@ def deactivate_drop(db_session: Session, dropper_claim_id: uuid.UUID): """ claim = ( - db_session.query(DropperClaim).filter(DropperClaim.id == dropper_claim_id).one() + db_session.query(DropperClaim).filter(DropperClaim.id == dropper_claim_id).one() # type: ignore ) claim.active = False @@ -300,7 +312,7 @@ def update_drop( """ claim = ( - db_session.query(DropperClaim).filter(DropperClaim.id == dropper_claim_id).one() + db_session.query(DropperClaim).filter(DropperClaim.id == dropper_claim_id).one() # type: ignore ) if title: @@ -619,7 +631,7 @@ def get_drop(db_session: Session, dropper_claim_id: uuid.UUID): Return particular drop """ drop = ( - db_session.query(DropperClaim).filter(DropperClaim.id == dropper_claim_id).one() + db_session.query(DropperClaim).filter(DropperClaim.id == dropper_claim_id).one() # type: ignore ) return drop @@ -833,7 +845,7 @@ def refetch_drop_signatures( ) .join(DropperContract, DropperClaim.dropper_contract_id == DropperContract.id) .filter(DropperClaim.id == dropper_claim_id) - ).one() + ).one() # type: ignore if claim.claim_block_deadline is None: raise DropWithNotSettedBlockDeadline( @@ -943,18 +955,79 @@ def get_leaderboard_total_count(db_session: Session, leaderboard_id): ) -def get_leaderboard(db_session: Session, leaderboard_id: uuid.UUID) -> Leaderboard: +def get_leaderboard_info(db_session: Session, leaderboard_id: uuid.UUID) -> Any: """ - Get the leaderboard from the database + Get the leaderboard from the database with users count """ leaderboard = ( - db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() + db_session.query( + Leaderboard.id, + Leaderboard.title, + Leaderboard.description, + func.count(LeaderboardScores.id).label("users_count"), + func.max(LeaderboardScores.updated_at).label("last_update"), + ) + .join(LeaderboardScores, LeaderboardScores.leaderboard_id == Leaderboard.id) + .filter(Leaderboard.id == leaderboard_id) + .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) + .one() ) return leaderboard +def get_leaderboard_scores_changes( + db_session: Session, leaderboard_id: uuid.UUID +) -> Any: + + """ + Return the leaderboard scores changes timeline changes of leaderboard scores + """ + + leaderboard_scores_changes = ( + db_session.query( + func.count(LeaderboardScores.address).label("players_count"), + # func.extract("epoch", LeaderboardScores.updated_at).label("timestamp"), + LeaderboardScores.updated_at.label("date"), + ) + .filter(LeaderboardScores.leaderboard_id == leaderboard_id) + .group_by(LeaderboardScores.updated_at) + .order_by(LeaderboardScores.updated_at.desc()) + ) + + return leaderboard_scores_changes + + +def get_leaderboard_scores_by_timestamp( + db_session: Session, + leaderboard_id: uuid.UUID, + date: datetime, + limit: int, + offset: int, +) -> Any: + + """ + Return the leaderboard scores by timestamp + """ + + leaderboard_scores = ( + db_session.query( + LeaderboardScores.leaderboard_id, + LeaderboardScores.address, + LeaderboardScores.score, + LeaderboardScores.points_data, + ) + .filter(LeaderboardScores.leaderboard_id == leaderboard_id) + .filter(LeaderboardScores.updated_at == date) + .order_by(LeaderboardScores.score.desc()) + .limit(limit) + .offset(offset) + ) + + return leaderboard_scores + + def get_leaderboards( db_session: Session, token: Union[str, uuid.UUID], @@ -1157,30 +1230,105 @@ def get_rank( return positions -def create_leaderboard(db_session: Session, title: str, description: str): +def create_leaderboard( + db_session: Session, + title: str, + description: Optional[str], + token: uuid.UUID, +): """ Create a leaderboard """ + try: + leaderboard = Leaderboard(title=title, description=description) + db_session.add(leaderboard) + db_session.commit() + + resource = create_leaderboard_resource( + leaderboard_id=str(leaderboard.id), + token=token, + ) + + leaderboard.resource_id = resource.id + + db_session.commit() + except Exception as e: + db_session.rollback() + logger.error(f"Error creating leaderboard: {e}") + raise LeaderboardCreateError(f"Error creating leaderboard: {e}") + + return leaderboard + + +def delete_leaderboard( + db_session: Session, leaderboard_id: uuid.UUID, token: uuid.UUID +): + + """ + Delete a leaderboard + """ + try: + leaderboard = ( + db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() # type: ignore + ) + + if leaderboard.resource_id is not None: + try: + bc.delete_resource( + token=token, + resource_id=leaderboard.resource_id, + ) + except Exception as e: + logger.error(f"Error deleting leaderboard resource: {e}") + + db_session.delete(leaderboard) + db_session.commit() + except Exception as e: + db_session.rollback() + logger.error(e) + raise LeaderboardDeleteError(f"Error deleting leaderboard: {e}") + + return leaderboard + + +def update_leaderboard( + db_session: Session, + leaderboard_id: uuid.UUID, + title: Optional[str], + description: Optional[str], + token: uuid.UUID, +): + + """ + Update a leaderboard + """ + + leaderboard = ( + db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() # type: ignore + ) + + if title is not None: + leaderboard.title = title + if description is not None: + leaderboard.description = description - leaderboard = Leaderboard(title=title, description=description) - db_session.add(leaderboard) db_session.commit() - return leaderboard.id + return leaderboard def get_leaderboard_by_id(db_session: Session, leaderboard_id): """ Get the leaderboard by id """ - return db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() + return db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() # type: ignore def get_leaderboard_by_title(db_session: Session, title): """ Get the leaderboard by title """ - return db_session.query(Leaderboard).filter(Leaderboard.title == title).one() + return db_session.query(Leaderboard).filter(Leaderboard.title == title).one() # type: ignore def list_leaderboards(db_session: Session, limit: int, offset: int): @@ -1265,8 +1413,10 @@ def add_scores( def create_leaderboard_resource( - leaderboard_id: uuid.UUID, - token: Optional[uuid.UUID] = None, + leaderboard_id: str, + token: Union[Optional[uuid.UUID], str] = None, + title: Optional[str] = None, + user_id: Optional[uuid.UUID] = None, ) -> BugoutResource: resource_data: Dict[str, Any] = { "type": LEADERBOARD_RESOURCE_TYPE, @@ -1275,19 +1425,22 @@ def create_leaderboard_resource( if token is None: token = MOONSTREAM_ADMIN_ACCESS_TOKEN - - resource = bc.create_resource( - token=MOONSTREAM_ADMIN_ACCESS_TOKEN, - application_id=MOONSTREAM_APPLICATION_ID, - resource_data=resource_data, - timeout=10, - ) + try: + resource = bc.create_resource( + token=token, + application_id=MOONSTREAM_APPLICATION_ID, + resource_data=resource_data, + timeout=10, + ) + except Exception as e: + raise LeaderboardCreateError(f"Error creating leaderboard resource: {e}") return resource def assign_resource( db_session: Session, leaderboard_id: uuid.UUID, + user_token: Union[uuid.UUID, str], resource_id: Optional[uuid.UUID] = None, ): """ @@ -1295,19 +1448,17 @@ def assign_resource( """ leaderboard = ( - db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() + db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() # type: ignore ) - if leaderboard.resource_id is not None: - raise Exception("Leaderboard already has a resource") - if resource_id is not None: leaderboard.resource_id = resource_id else: # Create resource via admin token resource = create_leaderboard_resource( - leaderboard_id=leaderboard_id, + leaderboard_id=str(leaderboard_id), + token=user_token, ) leaderboard.resource_id = resource.id @@ -1338,7 +1489,7 @@ def revoke_resource(db_session: Session, leaderboard_id: uuid.UUID): # TODO(ANDREY): Delete resource via admin token leaderboard = ( - db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() + db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() # type: ignore ) if leaderboard.resource_id is None: @@ -1359,7 +1510,7 @@ def check_leaderboard_resource_permissions( Check if the user has permissions to access the leaderboard """ leaderboard = ( - db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() + db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() # type: ignore ) permission_url = f"{bc.brood_url}/resources/{leaderboard.resource_id}/holders" diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index b192bd3b..cefcdf3a 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -355,3 +355,46 @@ class LeaderboardInfoResponse(BaseModel): id: UUID title: str description: Optional[str] = None + users_count: int + + +class LeaderboardCreateRequest(BaseModel): + title: str + description: Optional[str] = None + + +class LeaderboardCreatedResponse(BaseModel): + id: UUID + title: str + description: Optional[str] = None + resource_id: Optional[UUID] = None + created_at: datetime + updated_at: datetime + + +class LeaderboardUpdatedResponse(BaseModel): + id: UUID + title: str + description: Optional[str] = None + resource_id: Optional[UUID] = None + created_at: datetime + updated_at: datetime + + +class LeaderboardUpdateRequest(BaseModel): + title: Optional[str] = None + description: Optional[str] = None + + +class LeaderboardDeletedResponse(BaseModel): + id: UUID + title: str + description: Optional[str] = None + resource_id: Optional[UUID] = None + created_at: datetime + updated_at: datetime + + +class LeaderboardScoresChangesResponse(BaseModel): + players_count: int + date: datetime diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 648015fc..19df313e 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -1,6 +1,7 @@ """ Leaderboard API. """ +from datetime import datetime import logging from uuid import UUID @@ -37,6 +38,8 @@ leaderboad_whitelist = { "/leaderboard": "GET", "/leaderboard/rank": "GET", "/leaderboard/ranks": "GET", + "/leaderboard/docs": "GET", + "/leaderboard/openapi.json": "GET", } app = FastAPI( @@ -69,7 +72,7 @@ async def get_leadeboard( Returns leaderboard info. """ try: - leaderboard = actions.get_leaderboard(db_session, leaderboard_id) + leaderboard = actions.get_leaderboard_info(db_session, leaderboard_id) except NoResultFound as e: raise EngineHTTPException( status_code=404, @@ -83,6 +86,8 @@ async def get_leadeboard( id=leaderboard.id, title=leaderboard.title, description=leaderboard.description, + users_count=leaderboard.users_count, + last_updated=leaderboard.last_update, ) @@ -122,6 +127,35 @@ async def get_leaderboards( return results +@app.get("/scores/changes") +async def get_scores_changes( + leaderboard_id: UUID, + db_session: Session = Depends(db.yield_db_session), +) -> Any: + + """ + Returns the score history for the given address. + """ + + try: + + scores = actions.get_leaderboard_scores_changes(db_session, leaderboard_id) + except actions.LeaderboardIsEmpty: + raise EngineHTTPException(status_code=204, detail="Leaderboard is empty.") + + except Exception as e: + logger.error(f"Error while getting scores: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + return [ + data.LeaderboardScoresChangesResponse( + players_count=score.players_count, + date=score.date, + ) + for score in scores + ] + + @app.get("/count/addresses", response_model=data.CountAddressesResponse) async def count_addresses( leaderboard_id: UUID, @@ -271,6 +305,158 @@ async def leaderboard( return result +@app.post("", response_model=data.LeaderboardCreatedResponse) +@app.post("/", response_model=data.LeaderboardCreatedResponse) +async def create_leaderboard( + request: Request, + leaderboard: data.LeaderboardCreateRequest, + db_session: Session = Depends(db.yield_db_session), +) -> data.LeaderboardCreatedResponse: + + """ + + Create leaderboard. + """ + + token = request.state.token + + try: + created_leaderboard = actions.create_leaderboard( + db_session, + title=leaderboard.title, + description=leaderboard.description, + token=token, + ) + except actions.LeaderboardCreateError as e: + logger.error(f"Error while creating leaderboard: {e}") + raise EngineHTTPException( + status_code=500, + detail="Leaderboard creation failed. Please try again.", + ) + + except Exception as e: + logger.error(f"Error while creating leaderboard: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + # Add resource to the leaderboard + + return data.LeaderboardCreatedResponse( + id=created_leaderboard.id, + title=created_leaderboard.title, + description=created_leaderboard.description, + resource_id=created_leaderboard.resource_id, + created_at=created_leaderboard.created_at, + updated_at=created_leaderboard.updated_at, + ) + + +@app.put("/{leaderboard_id}", response_model=data.LeaderboardUpdatedResponse) +async def update_leaderboard( + request: Request, + leaderboard_id: UUID, + leaderboard: data.LeaderboardUpdateRequest, + db_session: Session = Depends(db.yield_db_session), +) -> data.LeaderboardUpdatedResponse: + + """ + + Update leaderboard. + """ + + token = request.state.token + + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=request.state.token, + ) + + if access != True: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard." + ) + + try: + updated_leaderboard = actions.update_leaderboard( + db_session=db_session, + leaderboard_id=leaderboard_id, + title=leaderboard.title, + description=leaderboard.description, + token=token, + ) + except actions.LeaderboardUpdateError as e: + logger.error(f"Error while updating leaderboard: {e}") + raise EngineHTTPException( + status_code=500, + detail="Leaderboard update failed. Please try again.", + ) + + except Exception as e: + logger.error(f"Error while updating leaderboard: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + return data.LeaderboardUpdatedResponse( + id=updated_leaderboard.id, + title=updated_leaderboard.title, + description=updated_leaderboard.description, + resource_id=updated_leaderboard.resource_id, + created_at=updated_leaderboard.created_at, + updated_at=updated_leaderboard.updated_at, + ) + + +@app.delete("/{leaderboard_id}", response_model=data.LeaderboardDeletedResponse) +async def delete_leaderboard( + request: Request, + leaderboard_id: UUID, + db_session: Session = Depends(db.yield_db_session), +) -> data.LeaderboardDeletedResponse: + + """ + + Delete leaderboard. + """ + + token = request.state.token + + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=request.state.token, + ) + + if access != True: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard." + ) + + try: + deleted_leaderboard = actions.delete_leaderboard( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=token, + ) + except actions.LeaderboardDeleteError as e: + logger.error(f"Error while deleting leaderboard: {e}") + raise EngineHTTPException( + status_code=500, + detail="Leaderboard deletion failed. Please try again.", + ) + + except Exception as e: + logger.error(f"Error while deleting leaderboard: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + return data.LeaderboardDeletedResponse( + id=deleted_leaderboard.id, + title=deleted_leaderboard.title, + description=deleted_leaderboard.description, + resource_id=deleted_leaderboard.resource_id, + created_at=deleted_leaderboard.created_at, + updated_at=deleted_leaderboard.updated_at, + ) + + @app.get("/rank", response_model=List[data.LeaderboardPosition]) async def rank( leaderboard_id: UUID, From 1d0f01cb09d501d8812eb916b7e205de23294b5d Mon Sep 17 00:00:00 2001 From: Andrey Date: Sun, 6 Aug 2023 17:06:18 +0300 Subject: [PATCH 23/50] Add changes. --- engineapi/engineapi/actions.py | 2 +- engineapi/engineapi/data.py | 1 + engineapi/engineapi/routes/leaderboard.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index e981a565..fe1fd7f9 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -968,7 +968,7 @@ def get_leaderboard_info(db_session: Session, leaderboard_id: uuid.UUID) -> Any: func.count(LeaderboardScores.id).label("users_count"), func.max(LeaderboardScores.updated_at).label("last_update"), ) - .join(LeaderboardScores, LeaderboardScores.leaderboard_id == Leaderboard.id) + .join(LeaderboardScores, LeaderboardScores.leaderboard_id == Leaderboard.id, isouter=True) .filter(Leaderboard.id == leaderboard_id) .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) .one() diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index cefcdf3a..4f5f889f 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -356,6 +356,7 @@ class LeaderboardInfoResponse(BaseModel): title: str description: Optional[str] = None users_count: int + last_updated_at: Optional[datetime] = None class LeaderboardCreateRequest(BaseModel): diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 19df313e..316c41de 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -87,7 +87,7 @@ async def get_leadeboard( title=leaderboard.title, description=leaderboard.description, users_count=leaderboard.users_count, - last_updated=leaderboard.last_update, + last_updated_at=leaderboard.last_update, ) From ba6fd3222f709f962ae80c5594420aa086fa54e5 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 7 Aug 2023 01:12:45 +0300 Subject: [PATCH 24/50] Change endpoints order. --- engineapi/engineapi/routes/leaderboard.py | 404 +++++++++++----------- 1 file changed, 203 insertions(+), 201 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 316c41de..85eca11e 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -36,6 +36,7 @@ leaderboad_whitelist = { "/leaderboard/count/addresses": "GET", "/leaderboard/position": "GET", "/leaderboard": "GET", + "/leaderboard/": "GET", "/leaderboard/rank": "GET", "/leaderboard/ranks": "GET", "/leaderboard/docs": "GET", @@ -63,207 +64,6 @@ app.add_middleware( ) -@app.get("/info", response_model=data.LeaderboardInfoResponse) -async def get_leadeboard( - leaderboard_id: UUID, - db_session: Session = Depends(db.yield_db_session), -) -> data.LeaderboardInfoResponse: - """ - Returns leaderboard info. - """ - try: - leaderboard = actions.get_leaderboard_info(db_session, leaderboard_id) - except NoResultFound as e: - raise EngineHTTPException( - status_code=404, - detail="Leaderboard not found.", - ) - except Exception as e: - logger.error(f"Error while getting leaderboard: {e}") - raise EngineHTTPException(status_code=500, detail="Internal server error") - - return data.LeaderboardInfoResponse( - id=leaderboard.id, - title=leaderboard.title, - description=leaderboard.description, - users_count=leaderboard.users_count, - last_updated_at=leaderboard.last_update, - ) - - -@app.get("/leaderboards", response_model=List[data.Leaderboard]) -async def get_leaderboards( - request: Request, db_session: Session = Depends(db.yield_db_session) -) -> List[data.Leaderboard]: - """ - Returns leaderboard list to which user has access. - """ - - token = request.state.token - - try: - leaderboards = actions.get_leaderboards(db_session, token) - except actions.LeaderboardsResourcesNotFound as e: - raise EngineHTTPException( - status_code=404, - detail="Leaderboards not found.", - ) - except Exception as e: - logger.error(f"Error while getting leaderboards: {e}") - raise EngineHTTPException(status_code=500, detail="Internal server error") - - results = [ - data.Leaderboard( - id=leaderboard.id, - title=leaderboard.title, - description=leaderboard.description, - resource_id=leaderboard.resource_id, - created_at=leaderboard.created_at, - updated_at=leaderboard.updated_at, - ) - for leaderboard in leaderboards - ] - - return results - - -@app.get("/scores/changes") -async def get_scores_changes( - leaderboard_id: UUID, - db_session: Session = Depends(db.yield_db_session), -) -> Any: - - """ - Returns the score history for the given address. - """ - - try: - - scores = actions.get_leaderboard_scores_changes(db_session, leaderboard_id) - except actions.LeaderboardIsEmpty: - raise EngineHTTPException(status_code=204, detail="Leaderboard is empty.") - - except Exception as e: - logger.error(f"Error while getting scores: {e}") - raise EngineHTTPException(status_code=500, detail="Internal server error") - - return [ - data.LeaderboardScoresChangesResponse( - players_count=score.players_count, - date=score.date, - ) - for score in scores - ] - - -@app.get("/count/addresses", response_model=data.CountAddressesResponse) -async def count_addresses( - leaderboard_id: UUID, - db_session: Session = Depends(db.yield_db_session), -) -> data.CountAddressesResponse: - """ - Returns the number of addresses in the leaderboard. - """ - - ### Check if leaderboard exists - try: - actions.get_leaderboard_by_id(db_session, leaderboard_id) - except NoResultFound as e: - raise EngineHTTPException( - status_code=404, - detail="Leaderboard not found.", - ) - except Exception as e: - logger.error(f"Error while getting leaderboard: {e}") - raise EngineHTTPException(status_code=500, detail="Internal server error") - - count = actions.get_leaderboard_total_count(db_session, leaderboard_id) - - return data.CountAddressesResponse(count=count) - - -@app.get("/quartiles", response_model=data.QuartilesResponse) -async def quartiles( - leaderboard_id: UUID, - db_session: Session = Depends(db.yield_db_session), -) -> data.QuartilesResponse: - """ - Returns the quartiles of the leaderboard. - """ - ### Check if leaderboard exists - try: - actions.get_leaderboard_by_id(db_session, leaderboard_id) - except NoResultFound as e: - raise EngineHTTPException( - status_code=404, - detail="Leaderboard not found.", - ) - except Exception as e: - logger.error(f"Error while getting leaderboard: {e}") - raise EngineHTTPException(status_code=500, detail="Internal server error") - - try: - q1, q2, q3 = actions.get_qurtiles(db_session, leaderboard_id) - - except actions.LeaderboardIsEmpty: - raise EngineHTTPException(status_code=204, detail="Leaderboard is empty.") - except Exception as e: - logger.error(f"Error while getting quartiles: {e}") - raise EngineHTTPException(status_code=500, detail="Internal server error") - - return data.QuartilesResponse( - percentile_25={"address": q1[0], "score": q1[1], "rank": q1[2]}, - percentile_50={"address": q2[0], "score": q2[1], "rank": q2[2]}, - percentile_75={"address": q3[0], "score": q3[1], "rank": q3[2]}, - ) - - -@app.get("/position", response_model=List[data.LeaderboardPosition]) -async def position( - leaderboard_id: UUID, - address: str, - window_size: int = 1, - limit: int = 10, - offset: int = 0, - normalize_addresses: bool = True, - db_session: Session = Depends(db.yield_db_session), -) -> List[data.LeaderboardPosition]: - """ - Returns the leaderboard posotion for the given address. - With given window size. - """ - - ### Check if leaderboard exists - try: - actions.get_leaderboard_by_id(db_session, leaderboard_id) - except NoResultFound as e: - raise EngineHTTPException( - status_code=404, - detail="Leaderboard not found.", - ) - except Exception as e: - logger.error(f"Error while getting leaderboard: {e}") - raise EngineHTTPException(status_code=500, detail="Internal server error") - - if normalize_addresses: - address = Web3.toChecksumAddress(address) - - positions = actions.get_position( - db_session, leaderboard_id, address, window_size, limit, offset - ) - - results = [ - data.LeaderboardPosition( - address=position.address, - score=position.score, - rank=position.rank, - points_data=position.points_data, - ) - for position in positions - ] - - return results - @app.get("", response_model=List[data.LeaderboardPosition]) @app.get("/", response_model=List[data.LeaderboardPosition]) @@ -457,6 +257,208 @@ async def delete_leaderboard( ) +@app.get("/count/addresses", response_model=data.CountAddressesResponse) +async def count_addresses( + leaderboard_id: UUID, + db_session: Session = Depends(db.yield_db_session), +) -> data.CountAddressesResponse: + """ + Returns the number of addresses in the leaderboard. + """ + + ### Check if leaderboard exists + try: + actions.get_leaderboard_by_id(db_session, leaderboard_id) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) + except Exception as e: + logger.error(f"Error while getting leaderboard: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + count = actions.get_leaderboard_total_count(db_session, leaderboard_id) + + return data.CountAddressesResponse(count=count) + + +@app.get("/info", response_model=data.LeaderboardInfoResponse) +async def get_leadeboard( + leaderboard_id: UUID, + db_session: Session = Depends(db.yield_db_session), +) -> data.LeaderboardInfoResponse: + """ + Returns leaderboard info. + """ + try: + leaderboard = actions.get_leaderboard_info(db_session, leaderboard_id) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) + except Exception as e: + logger.error(f"Error while getting leaderboard: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + return data.LeaderboardInfoResponse( + id=leaderboard.id, + title=leaderboard.title, + description=leaderboard.description, + users_count=leaderboard.users_count, + last_updated_at=leaderboard.last_update, + ) + + +@app.get("/leaderboards", response_model=List[data.Leaderboard]) +async def get_leaderboards( + request: Request, db_session: Session = Depends(db.yield_db_session) +) -> List[data.Leaderboard]: + """ + Returns leaderboard list to which user has access. + """ + + token = request.state.token + + try: + leaderboards = actions.get_leaderboards(db_session, token) + except actions.LeaderboardsResourcesNotFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboards not found.", + ) + except Exception as e: + logger.error(f"Error while getting leaderboards: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + results = [ + data.Leaderboard( + id=leaderboard.id, + title=leaderboard.title, + description=leaderboard.description, + resource_id=leaderboard.resource_id, + created_at=leaderboard.created_at, + updated_at=leaderboard.updated_at, + ) + for leaderboard in leaderboards + ] + + return results + + +@app.get("/scores/changes") +async def get_scores_changes( + leaderboard_id: UUID, + db_session: Session = Depends(db.yield_db_session), +) -> Any: + + """ + Returns the score history for the given address. + """ + + try: + + scores = actions.get_leaderboard_scores_changes(db_session, leaderboard_id) + except actions.LeaderboardIsEmpty: + raise EngineHTTPException(status_code=204, detail="Leaderboard is empty.") + + except Exception as e: + logger.error(f"Error while getting scores: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + return [ + data.LeaderboardScoresChangesResponse( + players_count=score.players_count, + date=score.date, + ) + for score in scores + ] + + +@app.get("/quartiles", response_model=data.QuartilesResponse) +async def quartiles( + leaderboard_id: UUID, + db_session: Session = Depends(db.yield_db_session), +) -> data.QuartilesResponse: + """ + Returns the quartiles of the leaderboard. + """ + ### Check if leaderboard exists + try: + actions.get_leaderboard_by_id(db_session, leaderboard_id) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) + except Exception as e: + logger.error(f"Error while getting leaderboard: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + try: + q1, q2, q3 = actions.get_qurtiles(db_session, leaderboard_id) + + except actions.LeaderboardIsEmpty: + raise EngineHTTPException(status_code=204, detail="Leaderboard is empty.") + except Exception as e: + logger.error(f"Error while getting quartiles: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + return data.QuartilesResponse( + percentile_25={"address": q1[0], "score": q1[1], "rank": q1[2]}, + percentile_50={"address": q2[0], "score": q2[1], "rank": q2[2]}, + percentile_75={"address": q3[0], "score": q3[1], "rank": q3[2]}, + ) + + +@app.get("/position", response_model=List[data.LeaderboardPosition]) +async def position( + leaderboard_id: UUID, + address: str, + window_size: int = 1, + limit: int = 10, + offset: int = 0, + normalize_addresses: bool = True, + db_session: Session = Depends(db.yield_db_session), +) -> List[data.LeaderboardPosition]: + """ + Returns the leaderboard posotion for the given address. + With given window size. + """ + + ### Check if leaderboard exists + try: + actions.get_leaderboard_by_id(db_session, leaderboard_id) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) + except Exception as e: + logger.error(f"Error while getting leaderboard: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + if normalize_addresses: + address = Web3.toChecksumAddress(address) + + positions = actions.get_position( + db_session, leaderboard_id, address, window_size, limit, offset + ) + + results = [ + data.LeaderboardPosition( + address=position.address, + score=position.score, + rank=position.rank, + points_data=position.points_data, + ) + for position in positions + ] + + return results + + @app.get("/rank", response_model=List[data.LeaderboardPosition]) async def rank( leaderboard_id: UUID, From c417fc14b623d3f0789cde7da453439de685dd78 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 7 Aug 2023 01:14:37 +0300 Subject: [PATCH 25/50] Whitelist changes endpoint. --- engineapi/engineapi/routes/leaderboard.py | 1 + 1 file changed, 1 insertion(+) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 85eca11e..4d43aabb 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -39,6 +39,7 @@ leaderboad_whitelist = { "/leaderboard/": "GET", "/leaderboard/rank": "GET", "/leaderboard/ranks": "GET", + "/scores/changes": "GET", "/leaderboard/docs": "GET", "/leaderboard/openapi.json": "GET", } From eab292e815f1991cce4d61f92091d345178f0cf0 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 7 Aug 2023 12:10:21 +0000 Subject: [PATCH 26/50] Holders to Requesters and no defaults, small fixes --- ...f_call_request_types_and_metatx_holders.py | 50 +++++++++---------- engineapi/engineapi/contracts_actions.py | 48 ++++++++++-------- engineapi/engineapi/data.py | 8 +-- engineapi/engineapi/models.py | 27 +++++----- engineapi/engineapi/routes/metatx.py | 14 +++--- 5 files changed, 75 insertions(+), 72 deletions(-) diff --git a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py index a2f69860..5771e8da 100644 --- a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py +++ b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py @@ -1,4 +1,4 @@ -"""Call request types and Metatx holders +"""Call request types and Metatx requesters Revision ID: b4257b10daaf Revises: dedd8a7d0624 @@ -38,8 +38,7 @@ def upgrade(): op.create_foreign_key(op.f('fk_registered_contracts_blockchain_id_blockchains'), 'registered_contracts', 'blockchains', ['blockchain_id'], ['id'], ondelete='CASCADE') # Manual - Start - op.execute(f"INSERT INTO blockchains (id, name, chain_id, testnet) VALUES ('{str(uuid.uuid4())}', 'polygon', 137, FALSE);") - op.execute(f"INSERT INTO blockchains (id, name, chain_id, testnet) VALUES ('{str(uuid.uuid4())}', 'mumbai', 80001, TRUE);") + op.execute(f"INSERT INTO blockchains (id, name, chain_id, testnet) VALUES ('{str(uuid.uuid4())}', 'ethereum', 1, FALSE),('{str(uuid.uuid4())}', 'polygon', 137, FALSE),('{str(uuid.uuid4())}', 'mumbai', 80001, TRUE),('{str(uuid.uuid4())}', 'wyrm', 322, FALSE),('{str(uuid.uuid4())}', 'zksync_era', 324, FALSE),('{str(uuid.uuid4())}', 'zksync_era_testnet', 280, TRUE),('{str(uuid.uuid4())}', 'gnosis', 100, FALSE);") op.execute("UPDATE registered_contracts SET blockchain_id = (SELECT id FROM blockchains WHERE blockchains.name = registered_contracts.blockchain);") op.alter_column("registered_contracts", "blockchain_id", nullable=False) # Manual - End @@ -63,8 +62,7 @@ def upgrade(): op.create_foreign_key(op.f('fk_call_requests_call_request_type_id_call_request_types'), 'call_requests', 'call_request_types', ['call_request_type_id'], ['id'], ondelete='CASCADE') # Manual - Start - op.execute(f"INSERT INTO call_request_types (id, request_type, description) VALUES ('{str(uuid.uuid4())}', 'raw', 'A generic smart contract. You can ask users to submit arbitrary calldata to this contract.');") - op.execute(f"INSERT INTO call_request_types (id, request_type, description) VALUES ('{str(uuid.uuid4())}', 'dropper-v0.2.0', 'A Dropper v0.2.0 contract. You can authorize users to submit claims against this contract.');") + op.execute(f"INSERT INTO call_request_types (id, request_type, description) VALUES ('{str(uuid.uuid4())}', 'raw', 'A generic smart contract. You can ask users to submit arbitrary calldata to this contract.'),('{str(uuid.uuid4())}', 'dropper-v0.2.0', 'A Dropper v0.2.0 contract. You can authorize users to submit claims against this contract.');") op.execute("UPDATE call_requests SET call_request_type_id = (SELECT call_request_types.id FROM call_request_types INNER JOIN registered_contracts ON call_requests.registered_contract_id = registered_contracts.id WHERE call_request_types.request_type = registered_contracts.contract_type);") op.alter_column("call_requests", "call_request_type_id", nullable=False) # Manual - End @@ -73,25 +71,25 @@ def upgrade(): op.drop_column('registered_contracts', 'contract_type') # Holders - op.create_table('metatx_holders', + op.create_table('metatx_requesters', sa.Column('id', sa.UUID(), nullable=False), sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), - sa.PrimaryKeyConstraint('id', name=op.f('pk_metatx_holders')), - sa.UniqueConstraint('id', name=op.f('uq_metatx_holders_id')) + sa.PrimaryKeyConstraint('id', name=op.f('pk_metatx_requesters')), + sa.UniqueConstraint('id', name=op.f('uq_metatx_requesters_id')) ) - op.add_column('call_requests', sa.Column('metatx_holder_id', sa.UUID(), nullable=True)) - op.create_foreign_key(op.f('fk_call_requests_metatx_holder_id_metatx_holders'), 'call_requests', 'metatx_holders', ['metatx_holder_id'], ['id'], ondelete='CASCADE') - op.add_column('registered_contracts', sa.Column('metatx_holder_id', sa.UUID(), nullable=True)) - op.create_foreign_key(op.f('fk_registered_contracts_metatx_holder_id_metatx_holders'), 'registered_contracts', 'metatx_holders', ['metatx_holder_id'], ['id'], ondelete='CASCADE') + op.add_column('call_requests', sa.Column('metatx_requester_id', sa.UUID(), nullable=True)) + op.create_foreign_key(op.f('fk_call_requests_metatx_requester_id_metatx_requesters'), 'call_requests', 'metatx_requesters', ['metatx_requester_id'], ['id'], ondelete='CASCADE') + op.add_column('registered_contracts', sa.Column('metatx_requester_id', sa.UUID(), nullable=True)) + op.create_foreign_key(op.f('fk_registered_contracts_metatx_requester_id_metatx_requesters'), 'registered_contracts', 'metatx_requesters', ['metatx_requester_id'], ['id'], ondelete='CASCADE') # Manual - Start - op.execute("INSERT INTO metatx_holders (id) SELECT DISTINCT moonstream_user_id FROM registered_contracts ON CONFLICT (id) DO NOTHING;") - op.execute("INSERT INTO metatx_holders (id) SELECT DISTINCT moonstream_user_id FROM call_requests ON CONFLICT (id) DO NOTHING;") - op.execute("UPDATE registered_contracts SET metatx_holder_id = moonstream_user_id;") - op.execute("UPDATE call_requests SET metatx_holder_id = moonstream_user_id;") - op.alter_column("call_requests", "metatx_holder_id", nullable=False) - op.alter_column("registered_contracts", "metatx_holder_id", nullable=False) + op.execute("INSERT INTO metatx_requesters (id) SELECT DISTINCT moonstream_user_id FROM registered_contracts ON CONFLICT (id) DO NOTHING;") + op.execute("INSERT INTO metatx_requesters (id) SELECT DISTINCT moonstream_user_id FROM call_requests ON CONFLICT (id) DO NOTHING;") + op.execute("UPDATE registered_contracts SET metatx_requester_id = moonstream_user_id;") + op.execute("UPDATE call_requests SET metatx_requester_id = moonstream_user_id;") + op.alter_column("call_requests", "metatx_requester_id", nullable=False) + op.alter_column("registered_contracts", "metatx_requester_id", nullable=False) # Manual - End op.drop_index('ix_call_requests_moonstream_user_id', table_name='call_requests') @@ -100,7 +98,7 @@ def upgrade(): op.drop_column('registered_contracts', 'moonstream_user_id') # Other - op.create_unique_constraint(op.f('uq_registered_contracts_blockchain_id'), 'registered_contracts', ['blockchain_id', 'metatx_holder_id', 'address']) + op.create_unique_constraint(op.f('uq_registered_contracts_blockchain_id'), 'registered_contracts', ['blockchain_id', 'metatx_requester_id', 'address']) op.create_unique_constraint(op.f('uq_call_requests_id'), 'call_requests', ['id']) op.create_unique_constraint(op.f('uq_registered_contracts_id'), 'registered_contracts', ['id']) @@ -152,18 +150,18 @@ def downgrade(): op.create_index('ix_call_requests_moonstream_user_id', 'call_requests', ['moonstream_user_id'], unique=False) # Manual - Start - op.execute("UPDATE registered_contracts SET moonstream_user_id = metatx_holder_id;") - op.execute("UPDATE call_requests SET moonstream_user_id = metatx_holder_id;") + op.execute("UPDATE registered_contracts SET moonstream_user_id = metatx_requester_id;") + op.execute("UPDATE call_requests SET moonstream_user_id = metatx_requester_id;") op.alter_column("registered_contracts", "moonstream_user_id", nullable=False) op.alter_column("call_requests", "moonstream_user_id", nullable=False) # Manual - End - op.drop_constraint(op.f('fk_registered_contracts_metatx_holder_id_metatx_holders'), 'registered_contracts', type_='foreignkey') - op.drop_column('registered_contracts', 'metatx_holder_id') - op.drop_constraint(op.f('fk_call_requests_metatx_holder_id_metatx_holders'), 'call_requests', type_='foreignkey') - op.drop_column('call_requests', 'metatx_holder_id') + op.drop_constraint(op.f('fk_registered_contracts_metatx_requester_id_metatx_requesters'), 'registered_contracts', type_='foreignkey') + op.drop_column('registered_contracts', 'metatx_requester_id') + op.drop_constraint(op.f('fk_call_requests_metatx_requester_id_metatx_requesters'), 'call_requests', type_='foreignkey') + op.drop_column('call_requests', 'metatx_requester_id') - op.drop_table('metatx_holders') + op.drop_table('metatx_requesters') # Other op.create_unique_constraint('uq_registered_contracts_blockchain', 'registered_contracts', ['blockchain', 'moonstream_user_id', 'address', 'contract_type']) diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index 5c7a8476..37b76c17 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -18,7 +18,7 @@ from .models import ( Blockchain, CallRequest, CallRequestType, - MetatxHolder, + MetatxRequester, RegisteredContract, ) @@ -55,7 +55,7 @@ def parse_registered_contract_response( id=obj[0].id, blockchain=obj[1].name, address=obj[0].address, - metatx_holder_id=obj[0].metatx_holder_id, + metatx_requester_id=obj[0].metatx_requester_id, title=obj[0].title, description=obj[0].description, image_uri=obj[0].image_uri, @@ -71,7 +71,7 @@ def parse_call_request_response( id=obj[0].id, contract_id=obj[0].registered_contract_id, contract_address=obj[1].address, - metatx_holder_id=obj[0].metatx_holder_id, + metatx_requester_id=obj[0].metatx_requester_id, call_request_type=obj[2].request_type, caller=obj[0].caller, method=obj[0].method, @@ -122,7 +122,7 @@ def register_contract( db_session: Session, blockchain_name: str, address: str, - metatx_holder_id: uuid.UUID, + metatx_requester_id: uuid.UUID, title: Optional[str], description: Optional[str], image_uri: Optional[str], @@ -139,14 +139,18 @@ def register_contract( if blockchain is None: raise UnsupportedBlockchain("Unsupported blockchain specified") - metatx_holder_stmt = insert(MetatxHolder.__table__).values(id=metatx_holder_id) - metatx_holder_stmt_do_nothing_stmt = metatx_holder_stmt.on_conflict_do_nothing() - db_session.execute(metatx_holder_stmt_do_nothing_stmt) + metatx_requester_stmt = insert(MetatxRequester.__table__).values( + id=metatx_requester_id + ) + metatx_requester_stmt_do_nothing_stmt = ( + metatx_requester_stmt.on_conflict_do_nothing() + ) + db_session.execute(metatx_requester_stmt_do_nothing_stmt) contract = RegisteredContract( blockchain_id=blockchain.id, address=Web3.toChecksumAddress(address), - metatx_holder_id=metatx_holder_id, + metatx_requester_id=metatx_requester_id, title=title, description=description, image_uri=image_uri, @@ -166,7 +170,7 @@ def register_contract( def update_registered_contract( db_session: Session, - metatx_holder_id: uuid.UUID, + metatx_requester_id: uuid.UUID, contract_id: uuid.UUID, title: Optional[str] = None, description: Optional[str] = None, @@ -174,7 +178,7 @@ def update_registered_contract( ignore_nulls: bool = True, ) -> Tuple[RegisteredContract, Blockchain]: """ - Update the registered contract with the given contract ID provided that the user with metatx_holder_id + Update the registered contract with the given contract ID provided that the user with metatx_requester_id has access to it. """ contract_with_blockchain = ( @@ -182,7 +186,7 @@ def update_registered_contract( .join(Blockchain, Blockchain.id == RegisteredContract.blockchain_id) .filter( RegisteredContract.id == contract_id, - RegisteredContract.metatx_holder_id == metatx_holder_id, + RegisteredContract.metatx_requester_id == metatx_requester_id, ) .one() ) @@ -210,7 +214,7 @@ def update_registered_contract( def get_registered_contract( db_session: Session, - metatx_holder_id: uuid.UUID, + metatx_requester_id: uuid.UUID, contract_id: uuid.UUID, ) -> Tuple[RegisteredContract, Blockchain]: """ @@ -219,7 +223,7 @@ def get_registered_contract( contract_with_blockchain = ( db_session.query(RegisteredContract, Blockchain) .join(Blockchain, Blockchain.id == RegisteredContract.blockchain_id) - .filter(RegisteredContract.metatx_holder_id == metatx_holder_id) + .filter(RegisteredContract.metatx_requester_id == metatx_requester_id) .filter(RegisteredContract.id == contract_id) .one() ) @@ -230,7 +234,7 @@ def get_registered_contract( def lookup_registered_contracts( db_session: Session, - metatx_holder_id: uuid.UUID, + metatx_requester_id: uuid.UUID, blockchain: Optional[str] = None, address: Optional[str] = None, limit: int = 10, @@ -242,7 +246,7 @@ def lookup_registered_contracts( query = ( db_session.query(RegisteredContract, Blockchain) .join(Blockchain, Blockchain.id == RegisteredContract.blockchain_id) - .filter(RegisteredContract.metatx_holder_id == metatx_holder_id) + .filter(RegisteredContract.metatx_requester_id == metatx_requester_id) ) if blockchain is not None: @@ -263,7 +267,7 @@ def lookup_registered_contracts( def delete_registered_contract( db_session: Session, - metatx_holder_id: uuid.UUID, + metatx_requester_id: uuid.UUID, registered_contract_id: uuid.UUID, ) -> Tuple[RegisteredContract, Blockchain]: """ @@ -273,7 +277,7 @@ def delete_registered_contract( contract_with_blockchain = ( db_session.query(RegisteredContract, Blockchain) .join(Blockchain, Blockchain.id == RegisteredContract.blockchain_id) - .filter(RegisteredContract.metatx_holder_id == metatx_holder_id) + .filter(RegisteredContract.metatx_requester_id == metatx_requester_id) .filter(RegisteredContract.id == registered_contract_id) .one() ) @@ -293,7 +297,7 @@ def delete_registered_contract( def request_calls( db_session: Session, - metatx_holder_id: uuid.UUID, + metatx_requester_id: uuid.UUID, registered_contract_id: Optional[uuid.UUID], contract_address: Optional[str], call_specs: List[data.CallSpecification], @@ -317,7 +321,7 @@ def request_calls( # Check that the moonstream_user_id matches a RegisteredContract with the given id or address query = db_session.query(RegisteredContract).filter( - RegisteredContract.metatx_holder_id == metatx_holder_id + RegisteredContract.metatx_requester_id == metatx_requester_id ) if registered_contract_id is not None: @@ -331,7 +335,7 @@ def request_calls( try: registered_contract = query.one() except NoResultFound: - raise ValueError("Invalid registered_contract_id or metatx_holder_id") + raise ValueError("Invalid registered_contract_id or metatx_requester_id") # Normalize the caller argument using Web3.toChecksumAddress contract_type = ContractType(registered_contract.contract_type) @@ -487,7 +491,7 @@ def list_call_requests( def delete_requests( db_session: Session, - metatx_holder_id: uuid.UUID, + metatx_requester_id: uuid.UUID, request_ids: List[uuid.UUID] = [], ) -> int: """ @@ -496,7 +500,7 @@ def delete_requests( try: requests_to_delete_query = ( db_session.query(CallRequest) - .filter(CallRequest.metatx_holder_id == metatx_holder_id) + .filter(CallRequest.metatx_requester_id == metatx_requester_id) .filter(CallRequest.id.in_(request_ids)) ) requests_to_delete_num: int = requests_to_delete_query.delete( diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 65be0006..b9f4dd61 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -254,14 +254,14 @@ class RegisteredContractResponse(BaseModel): id: UUID blockchain: Optional[str] = None address: str - metatx_holder_id: UUID + metatx_requester_id: UUID title: Optional[str] = None description: Optional[str] = None image_uri: Optional[str] = None created_at: datetime updated_at: datetime - @validator("id", "metatx_holder_id") + @validator("id", "metatx_requester_id") def validate_uuids(cls, v): return str(v) @@ -303,7 +303,7 @@ class CallRequestResponse(BaseModel): id: UUID contract_id: UUID contract_address: Optional[str] = None - metatx_holder_id: UUID + metatx_requester_id: UUID call_request_type: Optional[str] = None caller: str method: str @@ -315,7 +315,7 @@ class CallRequestResponse(BaseModel): class Config: orm_mode = True - @validator("id", "contract_id", "metatx_holder_id") + @validator("id", "contract_id", "metatx_requester_id") def validate_uuids(cls, v): return str(v) diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index 636c4e39..f750a8d2 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -181,17 +181,16 @@ class CallRequestType(Base): # type: ignore ) -class MetatxHolder(Base): # type: ignore +class MetatxRequester(Base): # type: ignore """ - MetatxHolder represents id of user from bugout authorization. + MetatxRequester represents id of user from bugout authorization. """ - __tablename__ = "metatx_holders" + __tablename__ = "metatx_requesters" id = Column( UUID(as_uuid=True), primary_key=True, - default=uuid.uuid4, unique=True, ) @@ -201,12 +200,12 @@ class MetatxHolder(Base): # type: ignore registered_contracts = relationship( "RegisteredContract", - back_populates="metatx_holder", + back_populates="metatx_requester", cascade="all, delete, delete-orphan", ) call_requests = relationship( "CallRequest", - back_populates="metatx_holder", + back_populates="metatx_requester", cascade="all, delete, delete-orphan", ) @@ -236,7 +235,7 @@ class RegisteredContract(Base): # type: ignore __table_args__ = ( UniqueConstraint( "blockchain_id", - "metatx_holder_id", + "metatx_requester_id", "address", ), ) @@ -247,9 +246,9 @@ class RegisteredContract(Base): # type: ignore default=uuid.uuid4, unique=True, ) - metatx_holder_id = Column( + metatx_requester_id = Column( UUID(as_uuid=True), - ForeignKey("metatx_holders.id", ondelete="CASCADE"), + ForeignKey("metatx_requesters.id", ondelete="CASCADE"), nullable=False, ) blockchain_id = Column( @@ -279,7 +278,9 @@ class RegisteredContract(Base): # type: ignore cascade="all, delete, delete-orphan", ) - metatx_holder = relationship("MetatxHolder", back_populates="registered_contracts") + metatx_requester = relationship( + "MetatxRequester", back_populates="registered_contracts" + ) blockchain = relationship("Blockchain", back_populates="registered_contracts") @@ -303,9 +304,9 @@ class CallRequest(Base): ForeignKey("call_request_types.id", ondelete="CASCADE"), nullable=False, ) - metatx_holder_id = Column( + metatx_requester_id = Column( UUID(as_uuid=True), - ForeignKey("metatx_holders.id", ondelete="CASCADE"), + ForeignKey("metatx_requesters.id", ondelete="CASCADE"), nullable=False, ) @@ -330,7 +331,7 @@ class CallRequest(Base): "RegisteredContract", back_populates="call_requests" ) call_request_type = relationship("CallRequestType", back_populates="call_requests") - metatx_holder = relationship("MetatxHolder", back_populates="call_requests") + metatx_requester = relationship("MetatxRequester", back_populates="call_requests") class Leaderboard(Base): # type: ignore diff --git a/engineapi/engineapi/routes/metatx.py b/engineapi/engineapi/routes/metatx.py index 35841266..02159000 100644 --- a/engineapi/engineapi/routes/metatx.py +++ b/engineapi/engineapi/routes/metatx.py @@ -103,7 +103,7 @@ async def list_registered_contracts_route( registered_contracts_with_blockchain = ( contracts_actions.lookup_registered_contracts( db_session=db_session, - metatx_holder_id=request.state.user.id, + metatx_requester_id=request.state.user.id, blockchain=blockchain, address=address, limit=limit, @@ -136,7 +136,7 @@ async def get_registered_contract_route( try: contract_with_blockchain = contracts_actions.get_registered_contract( db_session=db_session, - metatx_holder_id=request.state.user.id, + metatx_requester_id=request.state.user.id, contract_id=contract_id, ) except NoResultFound: @@ -167,7 +167,7 @@ async def register_contract_route( try: contract_with_blockchain = contracts_actions.register_contract( db_session=db_session, - metatx_holder_id=request.state.user.id, + metatx_requester_id=request.state.user.id, blockchain_name=contract.blockchain, address=contract.address, title=contract.title, @@ -206,7 +206,7 @@ async def update_contract_route( try: contract_with_blockchain = contracts_actions.update_registered_contract( db_session=db_session, - metatx_holder_id=request.state.user.id, + metatx_requester_id=request.state.user.id, contract_id=contract_id, title=update_info.title, description=update_info.description, @@ -243,7 +243,7 @@ async def delete_contract_route( try: deleted_contract_with_blockchain = contracts_actions.delete_registered_contract( db_session=db_session, - metatx_holder_id=request.state.user.id, + metatx_requester_id=request.state.user.id, registered_contract_id=contract_id, ) except Exception as err: @@ -356,7 +356,7 @@ async def create_requests( try: num_requests = contracts_actions.request_calls( db_session=db_session, - metatx_holder_id=request.state.user.id, + metatx_requester_id=request.state.user.id, registered_contract_id=data.contract_id, contract_address=data.contract_address, call_specs=data.specifications, @@ -386,7 +386,7 @@ async def delete_requests( try: deleted_requests = contracts_actions.delete_requests( db_session=db_session, - metatx_holder_id=request.state.user.id, + metatx_requester_id=request.state.user.id, request_ids=request_ids, ) except Exception as err: From 1bd2f3706be1c63d5b49895fb3b657f5ae63b038 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 7 Aug 2023 13:26:58 +0000 Subject: [PATCH 27/50] Call request method and required params in db, fixed validation --- ...f_call_request_types_and_metatx_holders.py | 7 +- engineapi/engineapi/contracts_actions.py | 101 +++++++++++------- engineapi/engineapi/data.py | 6 +- engineapi/engineapi/models.py | 4 +- engineapi/engineapi/routes/metatx.py | 15 +++ 5 files changed, 87 insertions(+), 46 deletions(-) diff --git a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py index 5771e8da..0b8403fe 100644 --- a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py +++ b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py @@ -9,6 +9,7 @@ import uuid from alembic import op import sqlalchemy as sa +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. @@ -52,6 +53,8 @@ def upgrade(): sa.Column('id', sa.UUID(), nullable=False), sa.Column('request_type', sa.VARCHAR(length=128), nullable=False), sa.Column('description', sa.String(), nullable=True), + sa.Column('required_params', postgresql.ARRAY(sa.String()), nullable=True), + sa.Column('method', sa.String(), nullable=True), sa.PrimaryKeyConstraint('id', name=op.f('pk_call_request_types')), sa.UniqueConstraint('id', name=op.f('uq_call_request_types_id')) ) @@ -62,9 +65,11 @@ def upgrade(): op.create_foreign_key(op.f('fk_call_requests_call_request_type_id_call_request_types'), 'call_requests', 'call_request_types', ['call_request_type_id'], ['id'], ondelete='CASCADE') # Manual - Start - op.execute(f"INSERT INTO call_request_types (id, request_type, description) VALUES ('{str(uuid.uuid4())}', 'raw', 'A generic smart contract. You can ask users to submit arbitrary calldata to this contract.'),('{str(uuid.uuid4())}', 'dropper-v0.2.0', 'A Dropper v0.2.0 contract. You can authorize users to submit claims against this contract.');") + op.execute(f"INSERT INTO call_request_types (id, request_type, description, required_params, method) VALUES ('{str(uuid.uuid4())}', 'raw', 'A generic smart contract. You can ask users to submit arbitrary calldata to this contract.',ARRAY ['calldata'],''),('{str(uuid.uuid4())}', 'dropper-v0.2.0', 'A Dropper v0.2.0 contract. You can authorize users to submit claims against this contract.',ARRAY ['dropId','requestID','blockDeadline','amount','signer','signature'],'claim');") op.execute("UPDATE call_requests SET call_request_type_id = (SELECT call_request_types.id FROM call_request_types INNER JOIN registered_contracts ON call_requests.registered_contract_id = registered_contracts.id WHERE call_request_types.request_type = registered_contracts.contract_type);") op.alter_column("call_requests", "call_request_type_id", nullable=False) + op.alter_column("call_request_types", "required_params", nullable=False) + op.alter_column("call_request_types", "method", nullable=False) # Manual - End op.drop_index('ix_registered_contracts_contract_type', table_name='registered_contracts') diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index 37b76c17..2dbe9d90 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -13,7 +13,6 @@ from sqlalchemy.orm import Session from web3 import Web3 from . import data, db -from .data import ContractType from .models import ( Blockchain, CallRequest, @@ -38,12 +37,30 @@ class InvalidAddressFormat(Exception): """ +class UnsupportedCallRequestType(Exception): + """ + Raised when unsupported call request type specified. + """ + + class UnsupportedBlockchain(Exception): """ Raised when unsupported blockchain specified. """ +class CallRequestMethodValueError(Exception): + """ + Raised when method not acceptable for specified request type. + """ + + +class CallRequestRequiredParamsValueError(Exception): + """ + Raised when required params not acceptable for specified request type. + """ + + class ContractAlreadyRegistered(Exception): pass @@ -83,39 +100,38 @@ def parse_call_request_response( def validate_method_and_params( - contract_type: ContractType, method: str, parameters: Dict[str, Any] -) -> None: + call_request_type: str, + method: str, + parameters: Dict[str, Any], + request_types: List[CallRequestType], +) -> CallRequestType: """ Validate the given method and parameters for the specified contract_type. """ - if contract_type == ContractType.raw: - if method != "": - raise ValueError("Method must be empty string for raw contract type") - if set(parameters.keys()) != {"calldata"}: - raise ValueError( - "Parameters must have only 'calldata' key for raw contract type" - ) - elif contract_type == ContractType.dropper: - if method != "claim": - raise ValueError("Method must be 'claim' for dropper contract type") - required_params = { - "dropId", - "requestID", - "blockDeadline", - "amount", - "signer", - "signature", - } - if set(parameters.keys()) != required_params: - raise ValueError( - f"Parameters must have {required_params} keys for dropper contract type" - ) - try: - Web3.toChecksumAddress(parameters["signer"]) - except: - raise InvalidAddressFormat("Parameter signer must be a valid address") - else: - raise ValueError(f"Unknown contract type {contract_type}") + for rt in request_types: + if rt.request_type == call_request_type: + if method != rt.method: + raise CallRequestMethodValueError( + f"Method must be {rt.method} for {rt.request_type} request type" + ) + + required_params_set = set(rt.required_params) + if set(parameters.keys()) != required_params_set: + raise CallRequestRequiredParamsValueError( + f"Parameters must have only {rt.required_params} key for {rt.request_type} request type" + ) + + if "signer" in parameters: + try: + Web3.toChecksumAddress(parameters["signer"]) + except: + raise InvalidAddressFormat( + "Parameter signer must be a valid address" + ) + + return rt + + raise UnsupportedCallRequestType(f"Unknown request type {call_request_type}") def register_contract( @@ -337,16 +353,26 @@ def request_calls( except NoResultFound: raise ValueError("Invalid registered_contract_id or metatx_requester_id") + request_types = db_session.query(CallRequestType).all() + # Normalize the caller argument using Web3.toChecksumAddress - contract_type = ContractType(registered_contract.contract_type) for specification in call_specs: normalized_caller = Web3.toChecksumAddress(specification.caller) # Validate the method and parameters for the contract_type try: - validate_method_and_params( - contract_type, specification.method, specification.parameters + call_request_type = validate_method_and_params( + specification.call_request_type, + specification.method, + specification.parameters, + request_types, ) + except UnsupportedCallRequestType as err: + raise UnsupportedCallRequestType(err) + except CallRequestMethodValueError as err: + raise CallRequestMethodValueError(err) + except CallRequestRequiredParamsValueError as err: + raise CallRequestRequiredParamsValueError(err) except InvalidAddressFormat as err: raise InvalidAddressFormat(err) except Exception as err: @@ -360,8 +386,9 @@ def request_calls( request = CallRequest( registered_contract_id=registered_contract.id, + call_request_type_id=call_request_type.id, + metatx_requester_id=metatx_requester_id, caller=normalized_caller, - moonstream_user_id=moonstream_user_id, method=specification.method, parameters=specification.parameters, expires_at=expires_at, @@ -661,8 +688,6 @@ def generate_cli() -> argparse.ArgumentParser: register_parser.add_argument( "-c", "--contract-type", - type=ContractType, - choices=ContractType, required=True, help="The type of the contract", ) @@ -722,8 +747,6 @@ def generate_cli() -> argparse.ArgumentParser: list_contracts_parser.add_argument( "-c", "--contract-type", - type=ContractType, - choices=ContractType, required=False, default=None, help="The type of the contract", diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index b9f4dd61..ccf75b43 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -203,11 +203,6 @@ class DropUpdatedResponse(BaseModel): active: bool = True -class ContractType(Enum): - raw = "raw" - dropper = "dropper-v0.2.0" - - class CallRequestTypeResponse(BaseModel): id: UUID request_type: str @@ -276,6 +271,7 @@ class RegisteredContractResponse(BaseModel): class CallSpecification(BaseModel): caller: str method: str + call_request_type: str parameters: Dict[str, Any] @validator("caller") diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index f750a8d2..edbcf420 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -13,7 +13,7 @@ from sqlalchemy import ( UniqueConstraint, Integer, ) -from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.dialects.postgresql import JSONB, UUID, ARRAY from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import relationship from sqlalchemy.ext.declarative import declarative_base @@ -173,6 +173,8 @@ class CallRequestType(Base): # type: ignore ) request_type = Column(VARCHAR(128), nullable=False, unique=True, index=True) description = Column(String, nullable=True) + required_params = Column(ARRAY(String), nullable=False) + method = Column(String, nullable=False) call_requests = relationship( "CallRequest", diff --git a/engineapi/engineapi/routes/metatx.py b/engineapi/engineapi/routes/metatx.py index 02159000..1837a339 100644 --- a/engineapi/engineapi/routes/metatx.py +++ b/engineapi/engineapi/routes/metatx.py @@ -367,6 +367,21 @@ async def create_requests( status_code=400, detail=f"Address not passed web3checksum validation, err: {err}", ) + except contracts_actions.UnsupportedCallRequestType as err: + raise EngineHTTPException( + status_code=400, + detail=f"Unsupported call request type specified, err: {err}", + ) + except contracts_actions.CallRequestMethodValueError as err: + raise EngineHTTPException( + status_code=400, + detail=f"Unacceptable call request method specified, err: {err}", + ) + except contracts_actions.CallRequestRequiredParamsValueError as err: + raise EngineHTTPException( + status_code=400, + detail=f"Unacceptable call request required params specified, err: {err}", + ) except Exception as err: logger.error(repr(err)) raise EngineHTTPException(status_code=500) From ba78712cc5d4123064cfd1e4f821c3a1796e5334 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 8 Aug 2023 12:17:49 +0000 Subject: [PATCH 28/50] Request type PK name and default set to dropper if not specified --- ...f_call_request_types_and_metatx_holders.py | 31 +++--- engineapi/engineapi/contracts_actions.py | 96 +++++++++---------- engineapi/engineapi/data.py | 5 +- engineapi/engineapi/models.py | 21 +--- 4 files changed, 67 insertions(+), 86 deletions(-) diff --git a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py index 0b8403fe..81b9c724 100644 --- a/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py +++ b/engineapi/alembic/versions/b4257b10daaf_call_request_types_and_metatx_holders.py @@ -50,26 +50,19 @@ def upgrade(): # Types op.create_table('call_request_types', - sa.Column('id', sa.UUID(), nullable=False), - sa.Column('request_type', sa.VARCHAR(length=128), nullable=False), + sa.Column('name', sa.VARCHAR(length=128), nullable=False), sa.Column('description', sa.String(), nullable=True), - sa.Column('required_params', postgresql.ARRAY(sa.String()), nullable=True), - sa.Column('method', sa.String(), nullable=True), - sa.PrimaryKeyConstraint('id', name=op.f('pk_call_request_types')), - sa.UniqueConstraint('id', name=op.f('uq_call_request_types_id')) + sa.PrimaryKeyConstraint('name', name=op.f('pk_call_request_types')), + sa.UniqueConstraint('name', name=op.f('uq_call_request_types_name')) ) - op.create_index(op.f('ix_call_request_types_request_type'), 'call_request_types', ['request_type'], unique=True) - - op.add_column('call_requests', sa.Column('call_request_type_id', sa.UUID(), nullable=True)) - op.create_foreign_key(op.f('fk_call_requests_call_request_type_id_call_request_types'), 'call_requests', 'call_request_types', ['call_request_type_id'], ['id'], ondelete='CASCADE') + op.add_column('call_requests', sa.Column('call_request_type_name', sa.VARCHAR(length=128), nullable=True)) + op.create_foreign_key(op.f('fk_call_requests_call_request_type_name_call_request_types'), 'call_requests', 'call_request_types', ['call_request_type_name'], ['name'], ondelete='CASCADE') # Manual - Start - op.execute(f"INSERT INTO call_request_types (id, request_type, description, required_params, method) VALUES ('{str(uuid.uuid4())}', 'raw', 'A generic smart contract. You can ask users to submit arbitrary calldata to this contract.',ARRAY ['calldata'],''),('{str(uuid.uuid4())}', 'dropper-v0.2.0', 'A Dropper v0.2.0 contract. You can authorize users to submit claims against this contract.',ARRAY ['dropId','requestID','blockDeadline','amount','signer','signature'],'claim');") - op.execute("UPDATE call_requests SET call_request_type_id = (SELECT call_request_types.id FROM call_request_types INNER JOIN registered_contracts ON call_requests.registered_contract_id = registered_contracts.id WHERE call_request_types.request_type = registered_contracts.contract_type);") - op.alter_column("call_requests", "call_request_type_id", nullable=False) - op.alter_column("call_request_types", "required_params", nullable=False) - op.alter_column("call_request_types", "method", nullable=False) + op.execute(f"INSERT INTO call_request_types (name, description) VALUES ('raw','A generic smart contract. You can ask users to submit arbitrary calldata to this contract.'),('dropper-v0.2.0','A Dropper v0.2.0 contract. You can authorize users to submit claims against this contract.');") + op.execute("UPDATE call_requests SET call_request_type_name = (SELECT call_request_types.name FROM call_request_types INNER JOIN registered_contracts ON call_requests.registered_contract_id = registered_contracts.id WHERE call_request_types.name = registered_contracts.contract_type);") + op.alter_column("call_requests", "call_request_type_name", nullable=False) # Manual - End op.drop_index('ix_registered_contracts_contract_type', table_name='registered_contracts') @@ -105,10 +98,10 @@ def upgrade(): # Other op.create_unique_constraint(op.f('uq_registered_contracts_blockchain_id'), 'registered_contracts', ['blockchain_id', 'metatx_requester_id', 'address']) - op.create_unique_constraint(op.f('uq_call_requests_id'), 'call_requests', ['id']) - op.create_unique_constraint(op.f('uq_registered_contracts_id'), 'registered_contracts', ['id']) - op.create_unique_constraint(op.f('uq_leaderboard_scores_id'), 'leaderboard_scores', ['id']) - op.create_unique_constraint(op.f('uq_leaderboards_id'), 'leaderboards', ['id']) + # op.create_unique_constraint(op.f('uq_call_requests_id'), 'call_requests', ['id']) + # op.create_unique_constraint(op.f('uq_registered_contracts_id'), 'registered_contracts', ['id']) + # op.create_unique_constraint(op.f('uq_leaderboard_scores_id'), 'leaderboard_scores', ['id']) + # op.create_unique_constraint(op.f('uq_leaderboards_id'), 'leaderboards', ['id']) # ### end Alembic commands ### diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index 2dbe9d90..d7255388 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -82,14 +82,14 @@ def parse_registered_contract_response( def parse_call_request_response( - obj: Tuple[CallRequest, RegisteredContract, CallRequestType, CallRequestType] + obj: Tuple[CallRequest, RegisteredContract] ) -> data.CallRequestResponse: return data.CallRequestResponse( id=obj[0].id, contract_id=obj[0].registered_contract_id, contract_address=obj[1].address, metatx_requester_id=obj[0].metatx_requester_id, - call_request_type=obj[2].request_type, + call_request_type=obj[0].call_request_type_name, caller=obj[0].caller, method=obj[0].method, parameters=obj[0].parameters, @@ -100,38 +100,48 @@ def parse_call_request_response( def validate_method_and_params( - call_request_type: str, - method: str, - parameters: Dict[str, Any], - request_types: List[CallRequestType], -) -> CallRequestType: + method: str, parameters: Dict[str, Any], call_request_type: Optional[str] = None +) -> str: """ Validate the given method and parameters for the specified contract_type. """ - for rt in request_types: - if rt.request_type == call_request_type: - if method != rt.method: - raise CallRequestMethodValueError( - f"Method must be {rt.method} for {rt.request_type} request type" - ) + if call_request_type is None or call_request_type == "dropper-v0.2.0": + call_request_type = "dropper-v0.2.0" + if method != "claim": + raise CallRequestMethodValueError( + "Method must be 'claim' for dropper contract type" + ) + required_params = { + "dropId", + "requestID", + "blockDeadline", + "amount", + "signer", + "signature", + } + if set(parameters.keys()) != required_params: + raise CallRequestRequiredParamsValueError( + f"Parameters must have {required_params} keys for dropper contract type" + ) + try: + Web3.toChecksumAddress(parameters["signer"]) + except: + raise InvalidAddressFormat("Parameter signer must be a valid address") - required_params_set = set(rt.required_params) - if set(parameters.keys()) != required_params_set: - raise CallRequestRequiredParamsValueError( - f"Parameters must have only {rt.required_params} key for {rt.request_type} request type" - ) + elif call_request_type == "raw": + if method != "": + raise CallRequestMethodValueError( + "Method must be empty string for raw contract type" + ) + if set(parameters.keys()) != {"calldata"}: + raise CallRequestRequiredParamsValueError( + "Parameters must have only 'calldata' key for raw contract type" + ) - if "signer" in parameters: - try: - Web3.toChecksumAddress(parameters["signer"]) - except: - raise InvalidAddressFormat( - "Parameter signer must be a valid address" - ) + else: + raise UnsupportedCallRequestType(f"Unknown contract type {call_request_type}") - return rt - - raise UnsupportedCallRequestType(f"Unknown request type {call_request_type}") + return call_request_type def register_contract( @@ -353,8 +363,6 @@ def request_calls( except NoResultFound: raise ValueError("Invalid registered_contract_id or metatx_requester_id") - request_types = db_session.query(CallRequestType).all() - # Normalize the caller argument using Web3.toChecksumAddress for specification in call_specs: normalized_caller = Web3.toChecksumAddress(specification.caller) @@ -362,10 +370,9 @@ def request_calls( # Validate the method and parameters for the contract_type try: call_request_type = validate_method_and_params( - specification.call_request_type, - specification.method, - specification.parameters, - request_types, + method=specification.method, + parameters=specification.parameters, + call_request_type=specification.call_request_type, ) except UnsupportedCallRequestType as err: raise UnsupportedCallRequestType(err) @@ -379,6 +386,7 @@ def request_calls( logger.error( f"Unhandled error occurred during methods and parameters validation, err: {err}" ) + raise Exception() expires_at = None if ttl_days is not None: @@ -386,7 +394,7 @@ def request_calls( request = CallRequest( registered_contract_id=registered_contract.id, - call_request_type_id=call_request_type.id, + call_request_type_name=call_request_type, metatx_requester_id=metatx_requester_id, caller=normalized_caller, method=specification.method, @@ -408,20 +416,16 @@ def request_calls( def get_call_requests( db_session: Session, request_id: uuid.UUID, -) -> Tuple[CallRequest, RegisteredContract, CallRequestType, CallRequestType]: +) -> Tuple[CallRequest, RegisteredContract]: """ Get call request by ID. """ results = ( - db_session.query(CallRequest, RegisteredContract, CallRequestType) + db_session.query(CallRequest, RegisteredContract) .join( RegisteredContract, CallRequest.registered_contract_id == RegisteredContract.id, ) - .join( - CallRequestType, - CallRequest.call_request_type_id == CallRequestType.id, - ) .filter(CallRequest.id == request_id) .all() ) @@ -432,9 +436,9 @@ def get_call_requests( f"Incorrect number of results found for request_id {request_id}" ) - call_request, registered_contract, call_request_type = results[0] + call_request, registered_contract = results[0] - return (call_request, registered_contract, call_request_type) + return (call_request, registered_contract) def list_blockchains( @@ -473,15 +477,11 @@ def list_call_requests( # If show_expired is False, filter out expired requests using current time on database server query = ( - db_session.query(CallRequest, RegisteredContract, CallRequestType) + db_session.query(CallRequest, RegisteredContract) .join( RegisteredContract, CallRequest.registered_contract_id == RegisteredContract.id, ) - .join( - CallRequestType, - CallRequest.call_request_type_id == CallRequestType.id, - ) .filter(CallRequest.caller == Web3.toChecksumAddress(caller)) ) diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index ccf75b43..78fa2abf 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -204,8 +204,7 @@ class DropUpdatedResponse(BaseModel): class CallRequestTypeResponse(BaseModel): - id: UUID - request_type: str + name: str description: str class Config: @@ -271,7 +270,7 @@ class RegisteredContractResponse(BaseModel): class CallSpecification(BaseModel): caller: str method: str - call_request_type: str + call_request_type: Optional[str] = None parameters: Dict[str, Any] @validator("caller") diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index edbcf420..895cb136 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -165,22 +165,12 @@ class CallRequestType(Base): # type: ignore __tablename__ = "call_request_types" - id = Column( - UUID(as_uuid=True), + name = Column( + VARCHAR(128), primary_key=True, - default=uuid.uuid4, unique=True, ) - request_type = Column(VARCHAR(128), nullable=False, unique=True, index=True) description = Column(String, nullable=True) - required_params = Column(ARRAY(String), nullable=False) - method = Column(String, nullable=False) - - call_requests = relationship( - "CallRequest", - back_populates="call_request_type", - cascade="all, delete, delete-orphan", - ) class MetatxRequester(Base): # type: ignore @@ -301,9 +291,9 @@ class CallRequest(Base): ForeignKey("registered_contracts.id", ondelete="CASCADE"), nullable=False, ) - call_request_type_id = Column( - UUID(as_uuid=True), - ForeignKey("call_request_types.id", ondelete="CASCADE"), + call_request_type_name = Column( + VARCHAR(128), + ForeignKey("call_request_types.name", ondelete="CASCADE"), nullable=False, ) metatx_requester_id = Column( @@ -332,7 +322,6 @@ class CallRequest(Base): registered_contract = relationship( "RegisteredContract", back_populates="call_requests" ) - call_request_type = relationship("CallRequestType", back_populates="call_requests") metatx_requester = relationship("MetatxRequester", back_populates="call_requests") From d4ff8e8870ae8992ae648fdc354ff641dd805dc6 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 8 Aug 2023 12:21:13 +0000 Subject: [PATCH 29/50] Fix default dropper value to be available from docs --- engineapi/engineapi/contracts_actions.py | 7 +++---- engineapi/engineapi/data.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index d7255388..e3039ff0 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -100,13 +100,12 @@ def parse_call_request_response( def validate_method_and_params( - method: str, parameters: Dict[str, Any], call_request_type: Optional[str] = None + call_request_type: str, method: str, parameters: Dict[str, Any] ) -> str: """ Validate the given method and parameters for the specified contract_type. """ - if call_request_type is None or call_request_type == "dropper-v0.2.0": - call_request_type = "dropper-v0.2.0" + if call_request_type == "dropper-v0.2.0": if method != "claim": raise CallRequestMethodValueError( "Method must be 'claim' for dropper contract type" @@ -370,9 +369,9 @@ def request_calls( # Validate the method and parameters for the contract_type try: call_request_type = validate_method_and_params( + call_request_type=specification.call_request_type, method=specification.method, parameters=specification.parameters, - call_request_type=specification.call_request_type, ) except UnsupportedCallRequestType as err: raise UnsupportedCallRequestType(err) diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 78fa2abf..fe90dbf0 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -270,7 +270,7 @@ class RegisteredContractResponse(BaseModel): class CallSpecification(BaseModel): caller: str method: str - call_request_type: Optional[str] = None + call_request_type: str = "dropper-v0.2.0" parameters: Dict[str, Any] @validator("caller") From 4ee6ce64063c61e7134374332b2551bbb2eb11cf Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 8 Aug 2023 19:05:13 +0000 Subject: [PATCH 30/50] Bumped version --- engineapi/engineapi/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engineapi/engineapi/version.txt b/engineapi/engineapi/version.txt index 81340c7e..bbdeab62 100644 --- a/engineapi/engineapi/version.txt +++ b/engineapi/engineapi/version.txt @@ -1 +1 @@ -0.0.4 +0.0.5 From 663c2be9d918551aee51afbdf08ad8c048c7293f Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 10 Aug 2023 10:40:46 +0000 Subject: [PATCH 31/50] Request ID for call requests migration --- .../040f2dfde5a5_request_id_bytes_column.py | 75 +++++++++++++++++++ engineapi/engineapi/models.py | 9 ++- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 engineapi/alembic/versions/040f2dfde5a5_request_id_bytes_column.py diff --git a/engineapi/alembic/versions/040f2dfde5a5_request_id_bytes_column.py b/engineapi/alembic/versions/040f2dfde5a5_request_id_bytes_column.py new file mode 100644 index 00000000..0f038f1e --- /dev/null +++ b/engineapi/alembic/versions/040f2dfde5a5_request_id_bytes_column.py @@ -0,0 +1,75 @@ +"""Request ID bytes column + +Revision ID: 040f2dfde5a5 +Revises: b4257b10daaf +Create Date: 2023-08-10 08:58:22.052336 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '040f2dfde5a5' +down_revision = 'b4257b10daaf' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('call_requests', sa.Column('request_id', sa.BigInteger(), nullable=True)) + op.create_index(op.f('ix_call_requests_request_id'), 'call_requests', ['request_id'], unique=False) + op.create_unique_constraint(op.f('uq_call_requests_registered_contract_id'), 'call_requests', ['registered_contract_id', 'request_id']) + + # Manual + # Fetch IDs of duplicates for 'dropper-v0.2.0' call_request_type and delete it + op.execute("""WITH Duplicates AS ( + SELECT + id, + registered_contract_id, + call_request_type_name, + parameters->'requestID' AS request_id, + created_at, + ROW_NUMBER() OVER ( + PARTITION BY + registered_contract_id, + call_request_type_name, + parameters->'requestID' + ORDER BY created_at ASC + ) AS row_num + FROM call_requests + WHERE call_request_type_name = 'dropper-v0.2.0' +), +DeleteDuplicates AS ( +SELECT id +FROM + Duplicates +WHERE + row_num < ( + SELECT COUNT(*) FROM Duplicates d2 + WHERE d2.registered_contract_id = Duplicates.registered_contract_id + AND d2.call_request_type_name = Duplicates.call_request_type_name + AND d2.request_id = Duplicates.request_id + ) +) +DELETE FROM call_requests WHERE id IN (SELECT id FROM DeleteDuplicates);""") + + # Fulfill not empty requestID values + op.execute("UPDATE call_requests SET request_id = CAST(parameters->>'requestID' AS bigint) WHERE parameters->>'requestID' IS NOT NULL;") + # Fulfill raw types with random requestID + op.execute("UPDATE call_requests SET request_id = FLOOR(RANDOM()* 120500600 + 120400600) WHERE parameters->>'requestID' IS NULL;") + op.alter_column("call_requests", "request_id", nullable=False) + + # Other + op.create_unique_constraint(op.f('uq_blockchains_id'), 'blockchains', ['id']) + op.create_unique_constraint(op.f('uq_call_request_types_name'), 'call_request_types', ['name']) + op.create_unique_constraint(op.f('uq_metatx_requesters_id'), 'metatx_requesters', ['id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_call_requests_request_id'), table_name='call_requests') + op.drop_column('call_requests', 'request_id') + # ### end Alembic commands ### diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index 895cb136..fa9d5743 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -13,7 +13,7 @@ from sqlalchemy import ( UniqueConstraint, Integer, ) -from sqlalchemy.dialects.postgresql import JSONB, UUID, ARRAY +from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import relationship from sqlalchemy.ext.declarative import declarative_base @@ -278,6 +278,12 @@ class RegisteredContract(Base): # type: ignore class CallRequest(Base): __tablename__ = "call_requests" + __table_args__ = ( + UniqueConstraint( + "registered_contract_id", + "request_id", + ), + ) id = Column( UUID(as_uuid=True), @@ -304,6 +310,7 @@ class CallRequest(Base): caller = Column(VARCHAR(256), nullable=False, index=True) method = Column(String, nullable=False, index=True) + request_id = Column(BigInteger, nullable=False, index=True) # TODO(zomglings): Should we conditional indices on parameters depending on the contract type? parameters = Column(JSONB, nullable=False) From f1a84d8cec27decec0a29ecc4aab9e036c19ce28 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 10 Aug 2023 10:54:53 +0000 Subject: [PATCH 32/50] Metatx routes fix to work with call requests --- engineapi/engineapi/contracts_actions.py | 12 +++++++++++- engineapi/engineapi/data.py | 2 ++ engineapi/engineapi/routes/metatx.py | 5 +++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index e3039ff0..8d9752cd 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -65,6 +65,12 @@ class ContractAlreadyRegistered(Exception): pass +class CallRequestAlreadyRegistered(Exception): + """ + Raised when call request with same parameters registered. + """ + + def parse_registered_contract_response( obj: Tuple[RegisteredContract, Blockchain] ) -> data.RegisteredContractResponse: @@ -92,6 +98,7 @@ def parse_call_request_response( call_request_type=obj[0].call_request_type_name, caller=obj[0].caller, method=obj[0].method, + request_id=obj[0].request_id, parameters=obj[0].parameters, expires_at=obj[0].expires_at, created_at=obj[0].created_at, @@ -112,7 +119,6 @@ def validate_method_and_params( ) required_params = { "dropId", - "requestID", "blockDeadline", "amount", "signer", @@ -397,6 +403,7 @@ def request_calls( metatx_requester_id=metatx_requester_id, caller=normalized_caller, method=specification.method, + request_id=specification.request_id, parameters=specification.parameters, expires_at=expires_at, ) @@ -405,6 +412,9 @@ def request_calls( # Insert the new rows into the database in a single transaction try: db_session.commit() + except IntegrityError as err: + db_session.rollback() + raise CallRequestAlreadyRegistered() except Exception as e: db_session.rollback() raise e diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index fe90dbf0..4e87ea8d 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -271,6 +271,7 @@ class CallSpecification(BaseModel): caller: str method: str call_request_type: str = "dropper-v0.2.0" + request_id: int parameters: Dict[str, Any] @validator("caller") @@ -302,6 +303,7 @@ class CallRequestResponse(BaseModel): call_request_type: Optional[str] = None caller: str method: str + request_id: int parameters: Dict[str, Any] expires_at: Optional[datetime] = None created_at: datetime diff --git a/engineapi/engineapi/routes/metatx.py b/engineapi/engineapi/routes/metatx.py index 1837a339..6ba090b1 100644 --- a/engineapi/engineapi/routes/metatx.py +++ b/engineapi/engineapi/routes/metatx.py @@ -382,6 +382,11 @@ async def create_requests( status_code=400, detail=f"Unacceptable call request required params specified, err: {err}", ) + except contracts_actions.CallRequestAlreadyRegistered: + raise EngineHTTPException( + status_code=409, + detail="Call request with same parameters registered", + ) except Exception as err: logger.error(repr(err)) raise EngineHTTPException(status_code=500) From 1ae366ab7d6adc804dc9d4b7aa75925a95be3347 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 10 Aug 2023 16:29:37 +0300 Subject: [PATCH 33/50] Update bugout version. --- crawlers/mooncrawl/mooncrawl/version.py | 2 +- crawlers/mooncrawl/setup.py | 2 +- moonstreamapi/moonstreamapi/version.py | 2 +- moonstreamapi/requirements.txt | 4 ++-- moonstreamapi/setup.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index 2d9a5a21..bcd791df 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.3.3" +MOONCRAWL_VERSION = "0.3.4" diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 2df7f7db..8a63a069 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -34,7 +34,7 @@ setup( zip_safe=False, install_requires=[ "boto3", - "bugout>=0.2.12", + "bugout>=0.2.13", "chardet", "fastapi", "moonstreamdb>=0.3.4", diff --git a/moonstreamapi/moonstreamapi/version.py b/moonstreamapi/moonstreamapi/version.py index 9f8fe571..5e53778c 100644 --- a/moonstreamapi/moonstreamapi/version.py +++ b/moonstreamapi/moonstreamapi/version.py @@ -2,4 +2,4 @@ Moonstream library and API version. """ -MOONSTREAMAPI_VERSION = "0.2.8" +MOONSTREAMAPI_VERSION = "0.2.9" diff --git a/moonstreamapi/requirements.txt b/moonstreamapi/requirements.txt index 73c5b18d..1c9c3764 100644 --- a/moonstreamapi/requirements.txt +++ b/moonstreamapi/requirements.txt @@ -9,7 +9,7 @@ base58==2.1.1 bitarray==2.6.0 boto3==1.26.5 botocore==1.29.5 -bugout>=0.2.12 +bugout>=0.2.13 certifi==2022.9.24 charset-normalizer==2.1.1 click==8.1.3 @@ -65,4 +65,4 @@ uvicorn==0.19.0 varint==1.0.2 web3==5.31.1 websockets==9.1 -yarl==1.8.1 \ No newline at end of file +yarl==1.8.1 diff --git a/moonstreamapi/setup.py b/moonstreamapi/setup.py index f58ea49f..2ee48301 100644 --- a/moonstreamapi/setup.py +++ b/moonstreamapi/setup.py @@ -13,7 +13,7 @@ setup( install_requires=[ "appdirs", "boto3", - "bugout>=0.2.12", + "bugout>=0.2.13", "fastapi", "moonstreamdb>=0.3.4", "humbug", From 9f376e86f75faeb085b2ca6bf4e1a591c125b51d Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 10 Aug 2023 16:33:46 +0300 Subject: [PATCH 34/50] Update on engine. --- engineapi/requirements.txt | 4 ++-- engineapi/setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/engineapi/requirements.txt b/engineapi/requirements.txt index 3a9a504b..1acebfe4 100644 --- a/engineapi/requirements.txt +++ b/engineapi/requirements.txt @@ -7,7 +7,7 @@ base58==2.1.1 bitarray==2.7.6 boto3==1.27.0 botocore==1.30.0 -bugout==0.2.9 +bugout==0.2.13 certifi==2023.5.7 charset-normalizer==3.1.0 click==8.1.3 @@ -61,4 +61,4 @@ uvicorn==0.22.0 varint==1.0.2 web3==5.31.4 websockets==9.1 -yarl==1.9.2 \ No newline at end of file +yarl==1.9.2 diff --git a/engineapi/setup.py b/engineapi/setup.py index e8db532a..20752874 100644 --- a/engineapi/setup.py +++ b/engineapi/setup.py @@ -13,7 +13,7 @@ setup( packages=find_packages(), install_requires=[ "boto3", - "bugout>=0.2.2", + "bugout>=0.2.13", "eip712==0.1.0", "eth-typing>=2.3.0", "fastapi", From 0c095b7696d13e871c08c8768bf6040f9103d4b9 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 10 Aug 2023 16:35:09 +0300 Subject: [PATCH 35/50] Add changes. --- engineapi/engineapi/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engineapi/engineapi/version.txt b/engineapi/engineapi/version.txt index 81340c7e..bbdeab62 100644 --- a/engineapi/engineapi/version.txt +++ b/engineapi/engineapi/version.txt @@ -1 +1 @@ -0.0.4 +0.0.5 From f15d70acd7c5882df66eb8ee7cb2773efdc9a9a8 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 10 Aug 2023 17:11:30 +0300 Subject: [PATCH 36/50] Refactor access check. --- engineapi/engineapi/routes/leaderboard.py | 169 +++++++++++++--------- 1 file changed, 102 insertions(+), 67 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 4d43aabb..60a779ee 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -160,17 +160,21 @@ async def update_leaderboard( ) -> data.LeaderboardUpdatedResponse: """ - Update leaderboard. """ token = request.state.token - - access = actions.check_leaderboard_resource_permissions( - db_session=db_session, - leaderboard_id=leaderboard_id, - token=request.state.token, - ) + try: + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=token, + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) if access != True: raise EngineHTTPException( @@ -214,17 +218,21 @@ async def delete_leaderboard( ) -> data.LeaderboardDeletedResponse: """ - Delete leaderboard. """ token = request.state.token - - access = actions.check_leaderboard_resource_permissions( - db_session=db_session, - leaderboard_id=leaderboard_id, - token=request.state.token, - ) + try: + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=token, + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) if access != True: raise EngineHTTPException( @@ -258,6 +266,75 @@ async def delete_leaderboard( ) +@app.get("/leaderboards", response_model=List[data.Leaderboard]) +async def get_leaderboards( + request: Request, db_session: Session = Depends(db.yield_db_session) +) -> List[data.Leaderboard]: + """ + Returns leaderboard list to which user has access. + """ + + token = request.state.token + + try: + leaderboards = actions.get_leaderboards(db_session, token) + except actions.LeaderboardsResourcesNotFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboards not found.", + ) + except Exception as e: + logger.error(f"Error while getting leaderboards: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + results = [ + data.Leaderboard( + id=leaderboard.id, + title=leaderboard.title, + description=leaderboard.description, + resource_id=leaderboard.resource_id, + created_at=leaderboard.created_at, + updated_at=leaderboard.updated_at, + ) + for leaderboard in leaderboards + ] + + return results + + +@app.get("/{leaderboard_id}/autoconfig", response_model=data.AutoConfigResponse) +async def autoconfig( + request: Request, + leaderboard_id: UUID, + db_session: Session = Depends(db.yield_db_session), +) -> data.AutoConfigResponse: + """ + Returns the autoconfig for the leaderboard. + """ + + token = request.state.token + try: + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=token, + ) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) + + if access != True: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard." + ) + + autoconfig = actions.get_autoconfig(db_session, leaderboard_id) + + return data.AutoConfigResponse(autoconfig=autoconfig) + + @app.get("/count/addresses", response_model=data.CountAddressesResponse) async def count_addresses( leaderboard_id: UUID, @@ -312,42 +389,6 @@ async def get_leadeboard( ) -@app.get("/leaderboards", response_model=List[data.Leaderboard]) -async def get_leaderboards( - request: Request, db_session: Session = Depends(db.yield_db_session) -) -> List[data.Leaderboard]: - """ - Returns leaderboard list to which user has access. - """ - - token = request.state.token - - try: - leaderboards = actions.get_leaderboards(db_session, token) - except actions.LeaderboardsResourcesNotFound as e: - raise EngineHTTPException( - status_code=404, - detail="Leaderboards not found.", - ) - except Exception as e: - logger.error(f"Error while getting leaderboards: {e}") - raise EngineHTTPException(status_code=500, detail="Internal server error") - - results = [ - data.Leaderboard( - id=leaderboard.id, - title=leaderboard.title, - description=leaderboard.description, - resource_id=leaderboard.resource_id, - created_at=leaderboard.created_at, - updated_at=leaderboard.updated_at, - ) - for leaderboard in leaderboards - ] - - return results - - @app.get("/scores/changes") async def get_scores_changes( leaderboard_id: UUID, @@ -543,29 +584,23 @@ async def leaderboard_push_scores( """ Put the leaderboard to the database. """ - - access = actions.check_leaderboard_resource_permissions( - db_session=db_session, - leaderboard_id=leaderboard_id, - token=request.state.token, - ) - - if not access: - raise EngineHTTPException( - status_code=403, detail="You don't have access to this leaderboard." - ) - - ### Check if leaderboard exists + token = request.state.token try: - actions.get_leaderboard_by_id(db_session, leaderboard_id) + access = actions.check_leaderboard_resource_permissions( + db_session=db_session, + leaderboard_id=leaderboard_id, + token=token, + ) except NoResultFound as e: raise EngineHTTPException( status_code=404, detail="Leaderboard not found.", ) - except Exception as e: - logger.error(f"Error while getting leaderboard: {e}") - raise EngineHTTPException(status_code=500, detail="Internal server error") + + if not access: + raise EngineHTTPException( + status_code=403, detail="You don't have access to this leaderboard." + ) try: leaderboard_points = actions.add_scores( From 20149770aa55f4b42af1dde3b1e83ea5c35eb71e Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 10 Aug 2023 18:24:26 +0300 Subject: [PATCH 37/50] Add changes. --- engineapi/requirements.txt | 2 +- engineapi/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engineapi/requirements.txt b/engineapi/requirements.txt index 1acebfe4..50a372ac 100644 --- a/engineapi/requirements.txt +++ b/engineapi/requirements.txt @@ -7,7 +7,7 @@ base58==2.1.1 bitarray==2.7.6 boto3==1.27.0 botocore==1.30.0 -bugout==0.2.13 +bugout==0.2.14 certifi==2023.5.7 charset-normalizer==3.1.0 click==8.1.3 diff --git a/engineapi/setup.py b/engineapi/setup.py index 20752874..d9f4e045 100644 --- a/engineapi/setup.py +++ b/engineapi/setup.py @@ -13,7 +13,7 @@ setup( packages=find_packages(), install_requires=[ "boto3", - "bugout>=0.2.13", + "bugout>=0.2.14", "eip712==0.1.0", "eth-typing>=2.3.0", "fastapi", From 5b38cf1effefacb907be9652a6b1f8e912b2c130 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 10 Aug 2023 18:26:26 +0300 Subject: [PATCH 38/50] Add changes. --- engineapi/engineapi/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engineapi/engineapi/version.txt b/engineapi/engineapi/version.txt index bbdeab62..1750564f 100644 --- a/engineapi/engineapi/version.txt +++ b/engineapi/engineapi/version.txt @@ -1 +1 @@ -0.0.5 +0.0.6 From 5e034737055be25e09c69f2d88b18397e9a5c111 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 10 Aug 2023 15:42:08 +0000 Subject: [PATCH 39/50] Switched call requests request id column to decimals --- ....py => 040f2dfde5a5_request_id_decimal_column.py} | 12 ++++++------ engineapi/engineapi/contracts_actions.py | 2 +- engineapi/engineapi/data.py | 4 ++-- engineapi/engineapi/models.py | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) rename engineapi/alembic/versions/{040f2dfde5a5_request_id_bytes_column.py => 040f2dfde5a5_request_id_decimal_column.py} (82%) diff --git a/engineapi/alembic/versions/040f2dfde5a5_request_id_bytes_column.py b/engineapi/alembic/versions/040f2dfde5a5_request_id_decimal_column.py similarity index 82% rename from engineapi/alembic/versions/040f2dfde5a5_request_id_bytes_column.py rename to engineapi/alembic/versions/040f2dfde5a5_request_id_decimal_column.py index 0f038f1e..f9315a8b 100644 --- a/engineapi/alembic/versions/040f2dfde5a5_request_id_bytes_column.py +++ b/engineapi/alembic/versions/040f2dfde5a5_request_id_decimal_column.py @@ -1,4 +1,4 @@ -"""Request ID bytes column +"""Request ID decimal column Revision ID: 040f2dfde5a5 Revises: b4257b10daaf @@ -18,7 +18,7 @@ depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column('call_requests', sa.Column('request_id', sa.BigInteger(), nullable=True)) + op.add_column('call_requests', sa.Column('request_id', sa.DECIMAL(), nullable=True)) op.create_index(op.f('ix_call_requests_request_id'), 'call_requests', ['request_id'], unique=False) op.create_unique_constraint(op.f('uq_call_requests_registered_contract_id'), 'call_requests', ['registered_contract_id', 'request_id']) @@ -56,15 +56,15 @@ WHERE DELETE FROM call_requests WHERE id IN (SELECT id FROM DeleteDuplicates);""") # Fulfill not empty requestID values - op.execute("UPDATE call_requests SET request_id = CAST(parameters->>'requestID' AS bigint) WHERE parameters->>'requestID' IS NOT NULL;") + op.execute("UPDATE call_requests SET request_id = CAST(parameters->>'requestID' AS DECIMAL) WHERE parameters->>'requestID' IS NOT NULL;") # Fulfill raw types with random requestID op.execute("UPDATE call_requests SET request_id = FLOOR(RANDOM()* 120500600 + 120400600) WHERE parameters->>'requestID' IS NULL;") op.alter_column("call_requests", "request_id", nullable=False) # Other - op.create_unique_constraint(op.f('uq_blockchains_id'), 'blockchains', ['id']) - op.create_unique_constraint(op.f('uq_call_request_types_name'), 'call_request_types', ['name']) - op.create_unique_constraint(op.f('uq_metatx_requesters_id'), 'metatx_requesters', ['id']) + # op.create_unique_constraint(op.f('uq_blockchains_id'), 'blockchains', ['id']) + # op.create_unique_constraint(op.f('uq_call_request_types_name'), 'call_request_types', ['name']) + # op.create_unique_constraint(op.f('uq_metatx_requesters_id'), 'metatx_requesters', ['id']) # ### end Alembic commands ### diff --git a/engineapi/engineapi/contracts_actions.py b/engineapi/engineapi/contracts_actions.py index 8d9752cd..e5a632c1 100644 --- a/engineapi/engineapi/contracts_actions.py +++ b/engineapi/engineapi/contracts_actions.py @@ -98,7 +98,7 @@ def parse_call_request_response( call_request_type=obj[0].call_request_type_name, caller=obj[0].caller, method=obj[0].method, - request_id=obj[0].request_id, + request_id=str(obj[0].request_id), parameters=obj[0].parameters, expires_at=obj[0].expires_at, created_at=obj[0].created_at, diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 4e87ea8d..3c7bdd48 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -271,7 +271,7 @@ class CallSpecification(BaseModel): caller: str method: str call_request_type: str = "dropper-v0.2.0" - request_id: int + request_id: str parameters: Dict[str, Any] @validator("caller") @@ -303,7 +303,7 @@ class CallRequestResponse(BaseModel): call_request_type: Optional[str] = None caller: str method: str - request_id: int + request_id: str parameters: Dict[str, Any] expires_at: Optional[datetime] = None created_at: datetime diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index fa9d5743..36cc975d 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -1,6 +1,7 @@ import uuid from sqlalchemy import ( + DECIMAL, VARCHAR, BigInteger, Boolean, @@ -8,15 +9,15 @@ from sqlalchemy import ( DateTime, ForeignKey, Index, + Integer, MetaData, String, UniqueConstraint, - Integer, ) from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.ext.compiler import compiles -from sqlalchemy.orm import relationship from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship from sqlalchemy.sql import and_, expression """ @@ -310,8 +311,7 @@ class CallRequest(Base): caller = Column(VARCHAR(256), nullable=False, index=True) method = Column(String, nullable=False, index=True) - request_id = Column(BigInteger, nullable=False, index=True) - # TODO(zomglings): Should we conditional indices on parameters depending on the contract type? + request_id = Column(DECIMAL, nullable=False, index=True) parameters = Column(JSONB, nullable=False) expires_at = Column(DateTime(timezone=True), nullable=True, index=True) From a7ab7063b04bbd8b12eef241ae99b0b1a4955676 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 10 Aug 2023 15:55:27 +0000 Subject: [PATCH 40/50] Clarified errors message for duplication of call request --- .../versions/040f2dfde5a5_request_id_decimal_column.py | 6 +++--- engineapi/engineapi/routes/metatx.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engineapi/alembic/versions/040f2dfde5a5_request_id_decimal_column.py b/engineapi/alembic/versions/040f2dfde5a5_request_id_decimal_column.py index f9315a8b..382f2074 100644 --- a/engineapi/alembic/versions/040f2dfde5a5_request_id_decimal_column.py +++ b/engineapi/alembic/versions/040f2dfde5a5_request_id_decimal_column.py @@ -62,9 +62,9 @@ DELETE FROM call_requests WHERE id IN (SELECT id FROM DeleteDuplicates);""") op.alter_column("call_requests", "request_id", nullable=False) # Other - # op.create_unique_constraint(op.f('uq_blockchains_id'), 'blockchains', ['id']) - # op.create_unique_constraint(op.f('uq_call_request_types_name'), 'call_request_types', ['name']) - # op.create_unique_constraint(op.f('uq_metatx_requesters_id'), 'metatx_requesters', ['id']) + op.create_unique_constraint(op.f('uq_blockchains_id'), 'blockchains', ['id']) + op.create_unique_constraint(op.f('uq_call_request_types_name'), 'call_request_types', ['name']) + op.create_unique_constraint(op.f('uq_metatx_requesters_id'), 'metatx_requesters', ['id']) # ### end Alembic commands ### diff --git a/engineapi/engineapi/routes/metatx.py b/engineapi/engineapi/routes/metatx.py index 6ba090b1..3af6bac9 100644 --- a/engineapi/engineapi/routes/metatx.py +++ b/engineapi/engineapi/routes/metatx.py @@ -385,7 +385,7 @@ async def create_requests( except contracts_actions.CallRequestAlreadyRegistered: raise EngineHTTPException( status_code=409, - detail="Call request with same parameters registered", + detail="Call request with same request_id already registered", ) except Exception as err: logger.error(repr(err)) From cb7950d92204d7f56dd2ec9633623eafa13b656a Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 10 Aug 2023 19:33:44 +0300 Subject: [PATCH 41/50] remove autoconfig endpoint. --- engineapi/engineapi/routes/leaderboard.py | 35 +---------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 60a779ee..582d18da 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -32,6 +32,7 @@ tags_metadata = [ leaderboad_whitelist = { "/leaderboard/info": "GET", + "/leaderboard/scores/changes": "GET", "/leaderboard/quartiles": "GET", "/leaderboard/count/addresses": "GET", "/leaderboard/position": "GET", @@ -301,40 +302,6 @@ async def get_leaderboards( return results - -@app.get("/{leaderboard_id}/autoconfig", response_model=data.AutoConfigResponse) -async def autoconfig( - request: Request, - leaderboard_id: UUID, - db_session: Session = Depends(db.yield_db_session), -) -> data.AutoConfigResponse: - """ - Returns the autoconfig for the leaderboard. - """ - - token = request.state.token - try: - access = actions.check_leaderboard_resource_permissions( - db_session=db_session, - leaderboard_id=leaderboard_id, - token=token, - ) - except NoResultFound as e: - raise EngineHTTPException( - status_code=404, - detail="Leaderboard not found.", - ) - - if access != True: - raise EngineHTTPException( - status_code=403, detail="You don't have access to this leaderboard." - ) - - autoconfig = actions.get_autoconfig(db_session, leaderboard_id) - - return data.AutoConfigResponse(autoconfig=autoconfig) - - @app.get("/count/addresses", response_model=data.CountAddressesResponse) async def count_addresses( leaderboard_id: UUID, From 3319d4a8a4af19920adf415c6b04fc076db69bcc Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 12 Aug 2023 09:14:33 +0300 Subject: [PATCH 42/50] Add type annotations. --- engineapi/engineapi/actions.py | 72 +++++++++++++---------- engineapi/engineapi/data.py | 6 ++ engineapi/engineapi/routes/leaderboard.py | 57 ++++++++---------- 3 files changed, 73 insertions(+), 62 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index fe1fd7f9..1ef43bc8 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1,6 +1,6 @@ from datetime import datetime from collections import Counter -from typing import List, Any, Optional, Dict, Union +from typing import List, Any, Optional, Dict, Union, Tuple import uuid import logging @@ -11,6 +11,7 @@ import requests # type: ignore from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session from sqlalchemy import func, text, or_ +from sqlalchemy.engine import Row from web3 import Web3 from web3.types import ChecksumAddress @@ -944,7 +945,7 @@ def refetch_drop_signatures( return claimant_objects -def get_leaderboard_total_count(db_session: Session, leaderboard_id): +def get_leaderboard_total_count(db_session: Session, leaderboard_id) -> int: """ Get the total number of claimants in the leaderboard """ @@ -955,7 +956,9 @@ def get_leaderboard_total_count(db_session: Session, leaderboard_id): ) -def get_leaderboard_info(db_session: Session, leaderboard_id: uuid.UUID) -> Any: +def get_leaderboard_info( + db_session: Session, leaderboard_id: uuid.UUID +) -> Row[Tuple[uuid.UUID, str, str, int, Optional[datetime]]]: """ Get the leaderboard from the database with users count """ @@ -968,7 +971,11 @@ def get_leaderboard_info(db_session: Session, leaderboard_id: uuid.UUID) -> Any: func.count(LeaderboardScores.id).label("users_count"), func.max(LeaderboardScores.updated_at).label("last_update"), ) - .join(LeaderboardScores, LeaderboardScores.leaderboard_id == Leaderboard.id, isouter=True) + .join( + LeaderboardScores, + LeaderboardScores.leaderboard_id == Leaderboard.id, + isouter=True, + ) .filter(Leaderboard.id == leaderboard_id) .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) .one() @@ -979,8 +986,7 @@ def get_leaderboard_info(db_session: Session, leaderboard_id: uuid.UUID) -> Any: def get_leaderboard_scores_changes( db_session: Session, leaderboard_id: uuid.UUID -) -> Any: - +) -> List[Row[Tuple[int, datetime]]]: """ Return the leaderboard scores changes timeline changes of leaderboard scores """ @@ -994,7 +1000,7 @@ def get_leaderboard_scores_changes( .filter(LeaderboardScores.leaderboard_id == leaderboard_id) .group_by(LeaderboardScores.updated_at) .order_by(LeaderboardScores.updated_at.desc()) - ) + ).all() return leaderboard_scores_changes @@ -1005,8 +1011,7 @@ def get_leaderboard_scores_by_timestamp( date: datetime, limit: int, offset: int, -) -> Any: - +) -> List[LeaderboardScores]: """ Return the leaderboard scores by timestamp """ @@ -1060,7 +1065,7 @@ def get_leaderboards( def get_position( db_session: Session, leaderboard_id, address, window_size, limit: int, offset: int -): +) -> List[Row[Tuple[str, int, int, int, Any]]]: """ Return position by address with window size @@ -1112,7 +1117,7 @@ def get_position( def get_leaderboard_positions( db_session: Session, leaderboard_id, limit: int, offset: int -): +) -> List[Row[Tuple[uuid.UUID, str, int, str, int]]]: """ Get the leaderboard positions """ @@ -1137,7 +1142,9 @@ def get_leaderboard_positions( return query -def get_qurtiles(db_session: Session, leaderboard_id): +def get_qurtiles( + db_session: Session, leaderboard_id +) -> Tuple[Row[Tuple[str, float, int]], ...]: """ Get the leaderboard qurtiles https://docs.sqlalchemy.org/en/14/core/functions.html#sqlalchemy.sql.functions.percentile_disc @@ -1171,7 +1178,7 @@ def get_qurtiles(db_session: Session, leaderboard_id): return q1, q2, q3 -def get_ranks(db_session: Session, leaderboard_id): +def get_ranks(db_session: Session, leaderboard_id) -> List[Row[Tuple[int, int, int]]]: """ Get the leaderboard rank buckets(rank, size, score) """ @@ -1199,7 +1206,7 @@ def get_rank( rank: int, limit: Optional[int] = None, offset: Optional[int] = None, -): +) -> List[Row[Tuple[uuid.UUID, str, int, str, int]]]: """ Get bucket in leaderboard by rank """ @@ -1234,11 +1241,14 @@ def create_leaderboard( db_session: Session, title: str, description: Optional[str], - token: uuid.UUID, -): + token: Optional[uuid.UUID], +) -> Leaderboard: """ Create a leaderboard """ + + if not token: + token = uuid.UUID(MOONSTREAM_ADMIN_ACCESS_TOKEN) try: leaderboard = Leaderboard(title=title, description=description) db_session.add(leaderboard) @@ -1262,8 +1272,7 @@ def create_leaderboard( def delete_leaderboard( db_session: Session, leaderboard_id: uuid.UUID, token: uuid.UUID -): - +) -> Leaderboard: """ Delete a leaderboard """ @@ -1280,6 +1289,10 @@ def delete_leaderboard( ) except Exception as e: logger.error(f"Error deleting leaderboard resource: {e}") + else: + logger.error( + f"Leaderboard {leaderboard_id} has no resource id. Skipping. Better delete it manually." + ) db_session.delete(leaderboard) db_session.commit() @@ -1296,9 +1309,7 @@ def update_leaderboard( leaderboard_id: uuid.UUID, title: Optional[str], description: Optional[str], - token: uuid.UUID, -): - +) -> Leaderboard: """ Update a leaderboard """ @@ -1317,21 +1328,23 @@ def update_leaderboard( return leaderboard -def get_leaderboard_by_id(db_session: Session, leaderboard_id): +def get_leaderboard_by_id(db_session: Session, leaderboard_id) -> Leaderboard: """ Get the leaderboard by id """ return db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() # type: ignore -def get_leaderboard_by_title(db_session: Session, title): +def get_leaderboard_by_title(db_session: Session, title) -> Leaderboard: """ Get the leaderboard by title """ return db_session.query(Leaderboard).filter(Leaderboard.title == title).one() # type: ignore -def list_leaderboards(db_session: Session, limit: int, offset: int): +def list_leaderboards( + db_session: Session, limit: int, offset: int +) -> List[Row[Tuple[uuid.UUID, str, str]]]: """ List all leaderboards """ @@ -1413,10 +1426,7 @@ def add_scores( def create_leaderboard_resource( - leaderboard_id: str, - token: Union[Optional[uuid.UUID], str] = None, - title: Optional[str] = None, - user_id: Optional[uuid.UUID] = None, + leaderboard_id: str, token: Union[Optional[uuid.UUID], str] = None ) -> BugoutResource: resource_data: Dict[str, Any] = { "type": LEADERBOARD_RESOURCE_TYPE, @@ -1481,7 +1491,9 @@ def list_leaderboards_resources( return query.all() -def revoke_resource(db_session: Session, leaderboard_id: uuid.UUID): +def revoke_resource( + db_session: Session, leaderboard_id: uuid.UUID +) -> Optional[uuid.UUID]: """ Revoke a resource handler to a leaderboard """ @@ -1505,7 +1517,7 @@ def revoke_resource(db_session: Session, leaderboard_id: uuid.UUID): def check_leaderboard_resource_permissions( db_session: Session, leaderboard_id: uuid.UUID, token: uuid.UUID -): +) -> bool: """ Check if the user has permissions to access the leaderboard """ diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 4f5f889f..fb3cb82f 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -372,6 +372,9 @@ class LeaderboardCreatedResponse(BaseModel): created_at: datetime updated_at: datetime + class Config: + orm_mode = True + class LeaderboardUpdatedResponse(BaseModel): id: UUID @@ -381,6 +384,9 @@ class LeaderboardUpdatedResponse(BaseModel): created_at: datetime updated_at: datetime + class Config: + orm_mode = True + class LeaderboardUpdateRequest(BaseModel): title: Optional[str] = None diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 582d18da..8df93685 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -66,7 +66,6 @@ app.add_middleware( ) - @app.get("", response_model=List[data.LeaderboardPosition]) @app.get("/", response_model=List[data.LeaderboardPosition]) async def leaderboard( @@ -114,7 +113,6 @@ async def create_leaderboard( leaderboard: data.LeaderboardCreateRequest, db_session: Session = Depends(db.yield_db_session), ) -> data.LeaderboardCreatedResponse: - """ Create leaderboard. @@ -143,12 +141,12 @@ async def create_leaderboard( # Add resource to the leaderboard return data.LeaderboardCreatedResponse( - id=created_leaderboard.id, - title=created_leaderboard.title, - description=created_leaderboard.description, - resource_id=created_leaderboard.resource_id, - created_at=created_leaderboard.created_at, - updated_at=created_leaderboard.updated_at, + id=created_leaderboard.id, # type: ignore + title=created_leaderboard.title, # type: ignore + description=created_leaderboard.description, # type: ignore + resource_id=created_leaderboard.resource_id, # type: ignore + created_at=created_leaderboard.created_at, # type: ignore + updated_at=created_leaderboard.updated_at, # type: ignore ) @@ -159,7 +157,6 @@ async def update_leaderboard( leaderboard: data.LeaderboardUpdateRequest, db_session: Session = Depends(db.yield_db_session), ) -> data.LeaderboardUpdatedResponse: - """ Update leaderboard. """ @@ -188,7 +185,6 @@ async def update_leaderboard( leaderboard_id=leaderboard_id, title=leaderboard.title, description=leaderboard.description, - token=token, ) except actions.LeaderboardUpdateError as e: logger.error(f"Error while updating leaderboard: {e}") @@ -202,12 +198,12 @@ async def update_leaderboard( raise EngineHTTPException(status_code=500, detail="Internal server error") return data.LeaderboardUpdatedResponse( - id=updated_leaderboard.id, - title=updated_leaderboard.title, - description=updated_leaderboard.description, - resource_id=updated_leaderboard.resource_id, - created_at=updated_leaderboard.created_at, - updated_at=updated_leaderboard.updated_at, + id=updated_leaderboard.id, # type: ignore + title=updated_leaderboard.title, # type: ignore + description=updated_leaderboard.description, # type: ignore + resource_id=updated_leaderboard.resource_id, # type: ignore + created_at=updated_leaderboard.created_at, # type: ignore + updated_at=updated_leaderboard.updated_at, # type: ignore ) @@ -217,7 +213,6 @@ async def delete_leaderboard( leaderboard_id: UUID, db_session: Session = Depends(db.yield_db_session), ) -> data.LeaderboardDeletedResponse: - """ Delete leaderboard. """ @@ -228,7 +223,7 @@ async def delete_leaderboard( db_session=db_session, leaderboard_id=leaderboard_id, token=token, - ) + ) except NoResultFound as e: raise EngineHTTPException( status_code=404, @@ -258,12 +253,11 @@ async def delete_leaderboard( raise EngineHTTPException(status_code=500, detail="Internal server error") return data.LeaderboardDeletedResponse( - id=deleted_leaderboard.id, - title=deleted_leaderboard.title, - description=deleted_leaderboard.description, - resource_id=deleted_leaderboard.resource_id, - created_at=deleted_leaderboard.created_at, - updated_at=deleted_leaderboard.updated_at, + id=deleted_leaderboard.id, # type: ignore + title=deleted_leaderboard.title, # type: ignore + description=deleted_leaderboard.description, # type: ignore + created_at=deleted_leaderboard.created_at, # type: ignore + updated_at=deleted_leaderboard.updated_at, # type: ignore ) @@ -290,18 +284,19 @@ async def get_leaderboards( results = [ data.Leaderboard( - id=leaderboard.id, - title=leaderboard.title, - description=leaderboard.description, - resource_id=leaderboard.resource_id, - created_at=leaderboard.created_at, - updated_at=leaderboard.updated_at, + id=leaderboard.id, # type: ignore + title=leaderboard.title, # type: ignore + description=leaderboard.description, # type: ignore + resource_id=leaderboard.resource_id, # type: ignore + created_at=leaderboard.created_at, # type: ignore + updated_at=leaderboard.updated_at, # type: ignore ) for leaderboard in leaderboards ] return results + @app.get("/count/addresses", response_model=data.CountAddressesResponse) async def count_addresses( leaderboard_id: UUID, @@ -361,13 +356,11 @@ async def get_scores_changes( leaderboard_id: UUID, db_session: Session = Depends(db.yield_db_session), ) -> Any: - """ Returns the score history for the given address. """ try: - scores = actions.get_leaderboard_scores_changes(db_session, leaderboard_id) except actions.LeaderboardIsEmpty: raise EngineHTTPException(status_code=204, detail="Leaderboard is empty.") From fbf106eba396683a65fc7483833ab9a642f6f46d Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 12 Aug 2023 09:18:19 +0300 Subject: [PATCH 43/50] Add changes. --- engineapi/engineapi/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 1ef43bc8..acc232e1 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1241,7 +1241,7 @@ def create_leaderboard( db_session: Session, title: str, description: Optional[str], - token: Optional[uuid.UUID], + token: Optional[Union[uuid.UUID, str]] = None, ) -> Leaderboard: """ Create a leaderboard From 9165a32f5efbcf039736add0e71df5e876bb97b0 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 12 Aug 2023 09:24:04 +0300 Subject: [PATCH 44/50] Add Query wrapper. --- engineapi/engineapi/routes/leaderboard.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 8df93685..ed843687 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -6,7 +6,7 @@ import logging from uuid import UUID from web3 import Web3 -from fastapi import FastAPI, Request, Depends, Response +from fastapi import FastAPI, Request, Depends, Response, Query from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from typing import Any, Dict, List, Optional @@ -70,8 +70,8 @@ app.add_middleware( @app.get("/", response_model=List[data.LeaderboardPosition]) async def leaderboard( leaderboard_id: UUID, - limit: int = 10, - offset: int = 0, + limit: int = Query(10), + offset: int = Query(0), db_session: Session = Depends(db.yield_db_session), ) -> List[data.LeaderboardPosition]: """ @@ -418,9 +418,9 @@ async def quartiles( async def position( leaderboard_id: UUID, address: str, - window_size: int = 1, - limit: int = 10, - offset: int = 0, + window_size: int = Query(1), + limit: int = Query(10), + offset: int = Query(0), normalize_addresses: bool = True, db_session: Session = Depends(db.yield_db_session), ) -> List[data.LeaderboardPosition]: @@ -464,9 +464,9 @@ async def position( @app.get("/rank", response_model=List[data.LeaderboardPosition]) async def rank( leaderboard_id: UUID, - rank: int = 1, - limit: Optional[int] = None, - offset: Optional[int] = None, + rank: int = Query(1), + limit: Optional[int] = Query(None), + offset: Optional[int] = Query(None), db_session: Session = Depends(db.yield_db_session), ) -> List[data.LeaderboardPosition]: """ From fa66d1cebe8f2d12afee7f9779f69b2fb618dd83 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 12 Aug 2023 09:48:38 +0300 Subject: [PATCH 45/50] Add descriptions and parameters types. --- engineapi/engineapi/routes/leaderboard.py | 54 ++++++++++++++--------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index ed843687..62056e53 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -6,7 +6,7 @@ import logging from uuid import UUID from web3 import Web3 -from fastapi import FastAPI, Request, Depends, Response, Query +from fastapi import FastAPI, Request, Depends, Response, Query, Path, Body from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from typing import Any, Dict, List, Optional @@ -69,7 +69,7 @@ app.add_middleware( @app.get("", response_model=List[data.LeaderboardPosition]) @app.get("/", response_model=List[data.LeaderboardPosition]) async def leaderboard( - leaderboard_id: UUID, + leaderboard_id: UUID = Query(..., description="Leaderboard ID"), limit: int = Query(10), offset: int = Query(0), db_session: Session = Depends(db.yield_db_session), @@ -110,7 +110,7 @@ async def leaderboard( @app.post("/", response_model=data.LeaderboardCreatedResponse) async def create_leaderboard( request: Request, - leaderboard: data.LeaderboardCreateRequest, + leaderboard: data.LeaderboardCreateRequest = Body(...), db_session: Session = Depends(db.yield_db_session), ) -> data.LeaderboardCreatedResponse: """ @@ -153,8 +153,8 @@ async def create_leaderboard( @app.put("/{leaderboard_id}", response_model=data.LeaderboardUpdatedResponse) async def update_leaderboard( request: Request, - leaderboard_id: UUID, - leaderboard: data.LeaderboardUpdateRequest, + leaderboard_id: UUID = Path(..., description="Leaderboard ID"), + leaderboard: data.LeaderboardUpdateRequest = Body(...), db_session: Session = Depends(db.yield_db_session), ) -> data.LeaderboardUpdatedResponse: """ @@ -210,7 +210,7 @@ async def update_leaderboard( @app.delete("/{leaderboard_id}", response_model=data.LeaderboardDeletedResponse) async def delete_leaderboard( request: Request, - leaderboard_id: UUID, + leaderboard_id: UUID = Path(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), ) -> data.LeaderboardDeletedResponse: """ @@ -299,7 +299,7 @@ async def get_leaderboards( @app.get("/count/addresses", response_model=data.CountAddressesResponse) async def count_addresses( - leaderboard_id: UUID, + leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), ) -> data.CountAddressesResponse: """ @@ -325,7 +325,7 @@ async def count_addresses( @app.get("/info", response_model=data.LeaderboardInfoResponse) async def get_leadeboard( - leaderboard_id: UUID, + leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), ) -> data.LeaderboardInfoResponse: """ @@ -353,9 +353,9 @@ async def get_leadeboard( @app.get("/scores/changes") async def get_scores_changes( - leaderboard_id: UUID, + leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), -) -> Any: +) -> List[data.LeaderboardScoresChangesResponse]: """ Returns the score history for the given address. """ @@ -380,7 +380,7 @@ async def get_scores_changes( @app.get("/quartiles", response_model=data.QuartilesResponse) async def quartiles( - leaderboard_id: UUID, + leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), ) -> data.QuartilesResponse: """ @@ -416,12 +416,14 @@ async def quartiles( @app.get("/position", response_model=List[data.LeaderboardPosition]) async def position( - leaderboard_id: UUID, - address: str, - window_size: int = Query(1), + leaderboard_id: UUID = Query(..., description="Leaderboard ID"), + address: str = Query(..., description="Address to get position for."), + window_size: int = Query(1, description="Amount of positions up and down."), limit: int = Query(10), offset: int = Query(0), - normalize_addresses: bool = True, + normalize_addresses: bool = Query( + True, description="Normalize addresses to checksum." + ), db_session: Session = Depends(db.yield_db_session), ) -> List[data.LeaderboardPosition]: """ @@ -463,8 +465,8 @@ async def position( @app.get("/rank", response_model=List[data.LeaderboardPosition]) async def rank( - leaderboard_id: UUID, - rank: int = Query(1), + leaderboard_id: UUID = Query(..., description="Leaderboard ID"), + rank: int = Query(1, description="Rank to get."), limit: Optional[int] = Query(None), offset: Optional[int] = Query(None), db_session: Session = Depends(db.yield_db_session), @@ -502,7 +504,8 @@ async def rank( @app.get("/ranks", response_model=List[data.RanksResponse]) async def ranks( - leaderboard_id: UUID, db_session: Session = Depends(db.yield_db_session) + leaderboard_id: UUID = Query(..., description="Leaderboard ID"), + db_session: Session = Depends(db.yield_db_session), ) -> List[data.RanksResponse]: """ Returns the leaderboard rank buckets overview with score and size of bucket. @@ -535,10 +538,17 @@ async def ranks( @app.put("/{leaderboard_id}/scores", response_model=List[data.LeaderboardScore]) async def leaderboard_push_scores( request: Request, - leaderboard_id: UUID, - scores: List[data.Score], - overwrite: bool = False, - normalize_addresses: bool = True, + leaderboard_id: UUID = Path(..., description="Leaderboard ID"), + scores: List[data.Score] = Body( + ..., description="Scores to put to the leaderboard." + ), + overwrite: bool = Query( + False, + description="If enabled, this will delete all current scores and replace them with the new scores provided.", + ), + normalize_addresses: bool = Query( + True, description="Normalize addresses to checksum." + ), db_session: Session = Depends(db.yield_db_session), ) -> List[data.LeaderboardScore]: """ From 3b52d8f49f43b60aa48d129e3ad8728a3ce6bfce Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 12 Aug 2023 09:54:46 +0300 Subject: [PATCH 46/50] Change endpoint name. --- engineapi/engineapi/routes/leaderboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 62056e53..c67467de 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -324,7 +324,7 @@ async def count_addresses( @app.get("/info", response_model=data.LeaderboardInfoResponse) -async def get_leadeboard( +async def leadeboard_info( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), ) -> data.LeaderboardInfoResponse: From fa92e3260df3c857864bd7e081d187e10b59645e Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 12 Aug 2023 11:13:51 +0300 Subject: [PATCH 47/50] Split endpoints by public and authorized. --- engineapi/engineapi/routes/leaderboard.py | 66 +++++++++++++++++------ 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index d5d95a73..d6fb2983 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -68,8 +68,8 @@ app.add_middleware( ) -@app.get("", response_model=List[data.LeaderboardPosition]) -@app.get("/", response_model=List[data.LeaderboardPosition]) +@app.get("", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"]) +@app.get("/", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"]) async def leaderboard( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), limit: int = Query(10), @@ -108,8 +108,12 @@ async def leaderboard( return result -@app.post("", response_model=data.LeaderboardCreatedResponse) -@app.post("/", response_model=data.LeaderboardCreatedResponse) +@app.post( + "", response_model=data.LeaderboardCreatedResponse, tags=["Authorized Endpoints"] +) +@app.post( + "/", response_model=data.LeaderboardCreatedResponse, tags=["Authorized Endpoints"] +) async def create_leaderboard( request: Request, leaderboard: data.LeaderboardCreateRequest = Body(...), @@ -152,7 +156,11 @@ async def create_leaderboard( ) -@app.put("/{leaderboard_id}", response_model=data.LeaderboardUpdatedResponse) +@app.put( + "/{leaderboard_id}", + response_model=data.LeaderboardUpdatedResponse, + tags=["Authorized Endpoints"], +) async def update_leaderboard( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), @@ -209,7 +217,11 @@ async def update_leaderboard( ) -@app.delete("/{leaderboard_id}", response_model=data.LeaderboardDeletedResponse) +@app.delete( + "/{leaderboard_id}", + response_model=data.LeaderboardDeletedResponse, + tags=["Authorized Endpoints"], +) async def delete_leaderboard( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), @@ -263,7 +275,11 @@ async def delete_leaderboard( ) -@app.get("/leaderboards", response_model=List[data.Leaderboard]) +@app.get( + "/leaderboards", + response_model=List[data.Leaderboard], + tags=["Authorized Endpoints"], +) async def get_leaderboards( request: Request, db_session: Session = Depends(db.yield_db_session) ) -> List[data.Leaderboard]: @@ -299,7 +315,11 @@ async def get_leaderboards( return results -@app.get("/count/addresses", response_model=data.CountAddressesResponse) +@app.get( + "/count/addresses", + response_model=data.CountAddressesResponse, + tags=["Public Endpoints"], +) async def count_addresses( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -325,7 +345,9 @@ async def count_addresses( return data.CountAddressesResponse(count=count) -@app.get("/info", response_model=data.LeaderboardInfoResponse) +@app.get( + "/info", response_model=data.LeaderboardInfoResponse, tags=["Public Endpoints"] +) async def leadeboard_info( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -353,7 +375,11 @@ async def leadeboard_info( ) -@app.get("/scores/changes") +@app.get( + "/scores/changes", + response_model=List[data.LeaderboardScoresChangesResponse], + tags=["Public Endpoints"], +) async def get_scores_changes( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -380,7 +406,7 @@ async def get_scores_changes( ] -@app.get("/quartiles", response_model=data.QuartilesResponse) +@app.get("/quartiles", response_model=data.QuartilesResponse, tags=["Public Endpoints"]) async def quartiles( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -416,7 +442,11 @@ async def quartiles( ) -@app.get("/position", response_model=List[data.LeaderboardPosition]) +@app.get( + "/position", + response_model=List[data.LeaderboardPosition], + tags=["Public Endpoints"], +) async def position( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), address: str = Query(..., description="Address to get position for."), @@ -465,7 +495,9 @@ async def position( return results -@app.get("/rank", response_model=List[data.LeaderboardPosition]) +@app.get( + "/rank", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"] +) async def rank( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), rank: int = Query(1, description="Rank to get."), @@ -504,7 +536,7 @@ async def rank( return results -@app.get("/ranks", response_model=List[data.RanksResponse]) +@app.get("/ranks", response_model=List[data.RanksResponse], tags=["Public Endpoints"]) async def ranks( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), @@ -537,7 +569,11 @@ async def ranks( return results -@app.put("/{leaderboard_id}/scores", response_model=List[data.LeaderboardScore]) +@app.put( + "/{leaderboard_id}/scores", + response_model=List[data.LeaderboardScore], + tags=["Authorized Endpoints"], +) async def leaderboard_push_scores( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), From b85cbb5f008d564e9f00736afe90c069a340b829 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 15 Aug 2023 13:55:13 +0300 Subject: [PATCH 48/50] Add Header parameter. --- engineapi/engineapi/routes/leaderboard.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index d6fb2983..4a4c8932 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -6,7 +6,7 @@ import logging from uuid import UUID from web3 import Web3 -from fastapi import FastAPI, Request, Depends, Response, Query, Path, Body +from fastapi import FastAPI, Request, Depends, Response, Query, Path, Body, Header from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound from typing import Any, Dict, List, Optional @@ -75,6 +75,9 @@ async def leaderboard( limit: int = Query(10), offset: int = Query(0), db_session: Session = Depends(db.yield_db_session), + Authorization: str = Header( + ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." + ), ) -> List[data.LeaderboardPosition]: """ Returns the leaderboard positions. @@ -118,6 +121,9 @@ async def create_leaderboard( request: Request, leaderboard: data.LeaderboardCreateRequest = Body(...), db_session: Session = Depends(db.yield_db_session), + Authorization: str = Header( + ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." + ), ) -> data.LeaderboardCreatedResponse: """ @@ -166,6 +172,9 @@ async def update_leaderboard( leaderboard_id: UUID = Path(..., description="Leaderboard ID"), leaderboard: data.LeaderboardUpdateRequest = Body(...), db_session: Session = Depends(db.yield_db_session), + Authorization: str = Header( + ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." + ), ) -> data.LeaderboardUpdatedResponse: """ Update leaderboard. @@ -226,6 +235,9 @@ async def delete_leaderboard( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), + Authorization: str = Header( + ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." + ), ) -> data.LeaderboardDeletedResponse: """ Delete leaderboard. @@ -281,7 +293,11 @@ async def delete_leaderboard( tags=["Authorized Endpoints"], ) async def get_leaderboards( - request: Request, db_session: Session = Depends(db.yield_db_session) + request: Request, + db_session: Session = Depends(db.yield_db_session), + Authorization: str = Header( + ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." + ), ) -> List[data.Leaderboard]: """ Returns leaderboard list to which user has access. @@ -588,6 +604,9 @@ async def leaderboard_push_scores( True, description="Normalize addresses to checksum." ), db_session: Session = Depends(db.yield_db_session), + Authorization: str = Header( + ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." + ), ) -> List[data.LeaderboardScore]: """ Put the leaderboard to the database. From 23d067071471543502f4a2f6eb21f741e514a46f Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 15 Aug 2023 19:04:08 +0300 Subject: [PATCH 49/50] Add changes. --- engineapi/engineapi/routes/leaderboard.py | 43 +++++++++++++---------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 4a4c8932..d2a5fbfa 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -26,9 +26,27 @@ logger = logging.getLogger(__name__) tags_metadata = [ - {"name": "leaderboard", "description": "Moonstream Engine leaderboard API"} + { + "name": "Public Endpoints", + "description": "Endpoints under this tag can be accessed without any authentication. They are open to all and do not require any specific headers or tokens to be passed. Suitable for general access and non-sensitive operations.", + }, + { + "name": "Authorized Endpoints", + "description": """ +Endpoints under this tag require authentication. To access these endpoints, a valid `moonstream token` must be included in the request header as: + +``` +Authorization: Bearer +``` + +Failure to provide a valid token will result in unauthorized access errors. These endpoints are suitable for operations that involve sensitive data or actions that only authenticated users are allowed to perform.""", + }, ] +AuthHeader = Header( + ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." +) + leaderboad_whitelist = { f"/leaderboard/{DOCS_TARGET_PATH}": "GET", @@ -75,9 +93,6 @@ async def leaderboard( limit: int = Query(10), offset: int = Query(0), db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), ) -> List[data.LeaderboardPosition]: """ Returns the leaderboard positions. @@ -121,9 +136,7 @@ async def create_leaderboard( request: Request, leaderboard: data.LeaderboardCreateRequest = Body(...), db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), + Authorization: str = AuthHeader, ) -> data.LeaderboardCreatedResponse: """ @@ -172,9 +185,7 @@ async def update_leaderboard( leaderboard_id: UUID = Path(..., description="Leaderboard ID"), leaderboard: data.LeaderboardUpdateRequest = Body(...), db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), + Authorization: str = AuthHeader, ) -> data.LeaderboardUpdatedResponse: """ Update leaderboard. @@ -235,9 +246,7 @@ async def delete_leaderboard( request: Request, leaderboard_id: UUID = Path(..., description="Leaderboard ID"), db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), + Authorization: str = AuthHeader, ) -> data.LeaderboardDeletedResponse: """ Delete leaderboard. @@ -295,9 +304,7 @@ async def delete_leaderboard( async def get_leaderboards( request: Request, db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), + Authorization: str = AuthHeader, ) -> List[data.Leaderboard]: """ Returns leaderboard list to which user has access. @@ -604,9 +611,7 @@ async def leaderboard_push_scores( True, description="Normalize addresses to checksum." ), db_session: Session = Depends(db.yield_db_session), - Authorization: str = Header( - ..., description="The expected format is 'Bearer YOUR_MOONSTREAM_ACCESS_TOKEN'." - ), + Authorization: str = AuthHeader, ) -> List[data.LeaderboardScore]: """ Put the leaderboard to the database. From a991549ea6c88e5eebed2586e58ef44d5b742c30 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 16 Aug 2023 05:21:44 +0300 Subject: [PATCH 50/50] Add status check of running task. --- crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py index 5c9f9da3..76549296 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -37,7 +37,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: ### get leaderboard journal - query = "#leaderboard" + query = "#leaderboard #status:active" if args.leaderboard_id: # way to run only one leaderboard query += f" #leaderboard_id:{args.leaderboard_id}" @@ -47,7 +47,7 @@ def handle_leaderboards(args: argparse.Namespace) -> None: journal_id=MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, query=query, limit=100, - timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, + timeout=10, ) leaderboards_results = cast(List[BugoutSearchResult], leaderboards.results) except Exception as e: