From 949f6c4905562bd87dbd2c8ce2c69be63a9923e3 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 9 Nov 2023 17:41:14 +0200 Subject: [PATCH 01/35] Add migration and endpoints changes. --- .../e86bb1230bca_leaderboard_metadata.py | 56 +++++++++++++++++++ engineapi/engineapi/actions.py | 27 ++++++++- engineapi/engineapi/data.py | 24 ++++++++ engineapi/engineapi/models.py | 5 ++ engineapi/engineapi/routes/leaderboard.py | 27 ++++++++- 5 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 engineapi/alembic/versions/e86bb1230bca_leaderboard_metadata.py diff --git a/engineapi/alembic/versions/e86bb1230bca_leaderboard_metadata.py b/engineapi/alembic/versions/e86bb1230bca_leaderboard_metadata.py new file mode 100644 index 00000000..3198bc75 --- /dev/null +++ b/engineapi/alembic/versions/e86bb1230bca_leaderboard_metadata.py @@ -0,0 +1,56 @@ +"""leaderboard metadata + +Revision ID: e86bb1230bca +Revises: 040f2dfde5a5 +Create Date: 2023-11-09 16:43:21.553490 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "e86bb1230bca" +down_revision = "040f2dfde5a5" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "leaderboards", + sa.Column( + "blockchain_ids", + sa.ARRAY(sa.Integer()), + nullable=False, + server_default="{}", + ), + ) + op.add_column( + "leaderboards", + sa.Column("show_connect", sa.Boolean(), nullable=False, server_default="false"), + ) + op.add_column( + "leaderboards", + sa.Column("public", sa.Boolean(), nullable=False, server_default="false"), + ) + op.add_column( + "leaderboards", + sa.Column( + "columns_names", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default="{}", + ), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("leaderboards", "columns_names") + op.drop_column("leaderboards", "public") + op.drop_column("leaderboards", "show_connect") + op.drop_column("leaderboards", "blockchain_ids") + # ### end Alembic commands ### diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 23c3e20f..feb5fe8d 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1256,6 +1256,10 @@ def create_leaderboard( title: str, description: Optional[str], token: Optional[Union[uuid.UUID, str]] = None, + public: bool = False, + show_connect: bool = False, + blockchain_ids: Optional[List[int]] = None, + columns_names: Optional[Dict[str, str]] = None, ) -> Leaderboard: """ Create a leaderboard @@ -1264,7 +1268,14 @@ def create_leaderboard( if not token: token = uuid.UUID(MOONSTREAM_ADMIN_ACCESS_TOKEN) try: - leaderboard = Leaderboard(title=title, description=description) + leaderboard = Leaderboard( + title=title, + description=description, + public=public, + show_connect=show_connect, + blockchain_ids=blockchain_ids, + columns_names=columns_names, + ) db_session.add(leaderboard) db_session.commit() @@ -1323,6 +1334,10 @@ def update_leaderboard( leaderboard_id: uuid.UUID, title: Optional[str], description: Optional[str], + public: Optional[bool], + show_connect: Optional[bool], + blockchain_ids: Optional[List[int]], + columns_names: Optional[Dict[str, str]], ) -> Leaderboard: """ Update a leaderboard @@ -1336,6 +1351,14 @@ def update_leaderboard( leaderboard.title = title if description is not None: leaderboard.description = description + if public is not None: + leaderboard.public = public + if show_connect is not None: + leaderboard.show_connect = show_connect + if blockchain_ids is not None: + leaderboard.blockchain_ids = blockchain_ids + if columns_names is not None: + leaderboard.columns_names = columns_names db_session.commit() @@ -1346,7 +1369,7 @@ def get_leaderboard_by_id(db_session: Session, leaderboard_id) -> Leaderboard: """ Get the leaderboard by id """ - return db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() # type: ignore + return db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).filter(leaderboard.public == True).one() # type: ignore def get_leaderboard_by_title(db_session: Session, title) -> Leaderboard: diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 8a33d619..49fa92be 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -370,6 +370,10 @@ class Leaderboard(BaseModel): title: str description: Optional[str] = None resource_id: Optional[UUID] = None + public: Optional[bool] = False + show_connect: Optional[bool] = False + blockchain_ids: Optional[List[int]] = None + columns_names: Optional[Dict[str, str]] = None created_at: datetime updated_at: datetime @@ -385,6 +389,10 @@ class LeaderboardInfoResponse(BaseModel): class LeaderboardCreateRequest(BaseModel): title: str description: Optional[str] = None + public: Optional[bool] = False + show_connect: Optional[bool] = False + blockchain_ids: Optional[List[int]] = None + columns_names: Optional[Dict[str, str]] = None class LeaderboardCreatedResponse(BaseModel): @@ -392,6 +400,10 @@ class LeaderboardCreatedResponse(BaseModel): title: str description: Optional[str] = None resource_id: Optional[UUID] = None + public: Optional[bool] = False + show_connect: Optional[bool] = False + blockchain_ids: Optional[List[int]] = None + columns_names: Optional[Dict[str, str]] = None created_at: datetime updated_at: datetime @@ -404,6 +416,10 @@ class LeaderboardUpdatedResponse(BaseModel): title: str description: Optional[str] = None resource_id: Optional[UUID] = None + public: Optional[bool] = False + show_connect: Optional[bool] = False + blockchain_ids: Optional[List[int]] = None + columns_names: Optional[Dict[str, str]] = None created_at: datetime updated_at: datetime @@ -414,6 +430,10 @@ class LeaderboardUpdatedResponse(BaseModel): class LeaderboardUpdateRequest(BaseModel): title: Optional[str] = None description: Optional[str] = None + public: Optional[bool] = False + show_connect: Optional[bool] = False + blockchain_ids: Optional[List[int]] = None + columns_names: Optional[Dict[str, str]] = None class LeaderboardDeletedResponse(BaseModel): @@ -421,6 +441,10 @@ class LeaderboardDeletedResponse(BaseModel): title: str description: Optional[str] = None resource_id: Optional[UUID] = None + public: Optional[bool] = False + show_connect: Optional[bool] = False + blockchain_ids: Optional[List[int]] = None + columns_names: Optional[Dict[str, str]] = None created_at: datetime updated_at: datetime diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index 36cc975d..a937fe07 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -1,6 +1,7 @@ import uuid from sqlalchemy import ( + ARRAY, DECIMAL, VARCHAR, BigInteger, @@ -346,6 +347,10 @@ class Leaderboard(Base): # type: ignore title = Column(VARCHAR(128), nullable=False) description = Column(String, nullable=True) resource_id = Column(UUID(as_uuid=True), nullable=True, index=True) + blockchain_ids = Column(ARRAY(Integer), nullable=False, default=[]) + show_connect = Column(Boolean, default=False, nullable=False) + public = Column(Boolean, default=False, nullable=False) + columns_names = Column(JSONB, nullable=False, default={}) created_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False ) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 15050f31..fbbbbb08 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -114,6 +114,7 @@ async def leaderboard( leaderboard_positions = actions.get_leaderboard_positions( db_session, leaderboard_id, limit, offset ) + result = [ data.LeaderboardPosition( address=position.address, @@ -140,7 +141,6 @@ async def create_leaderboard( Authorization: str = AuthHeader, ) -> data.LeaderboardCreatedResponse: """ - Create leaderboard. """ @@ -152,6 +152,10 @@ async def create_leaderboard( title=leaderboard.title, description=leaderboard.description, token=token, + public=leaderboard.public, + show_connect=leaderboard.show_connect, + blockchain_ids=leaderboard.blockchain_ids, + columns_names=leaderboard.columns_names, ) except actions.LeaderboardCreateError as e: logger.error(f"Error while creating leaderboard: {e}") @@ -171,6 +175,10 @@ async def create_leaderboard( title=created_leaderboard.title, # type: ignore description=created_leaderboard.description, # type: ignore resource_id=created_leaderboard.resource_id, # type: ignore + public=created_leaderboard.public, # type: ignore + show_connect=created_leaderboard.show_connect, # type: ignore + blockchain_ids=created_leaderboard.blockchain_ids, # type: ignore + columns_names=created_leaderboard.columns_names, # type: ignore created_at=created_leaderboard.created_at, # type: ignore updated_at=created_leaderboard.updated_at, # type: ignore ) @@ -216,6 +224,10 @@ async def update_leaderboard( leaderboard_id=leaderboard_id, title=leaderboard.title, description=leaderboard.description, + public=leaderboard.public, + show_connect=leaderboard.show_connect, + blockchain_ids=leaderboard.blockchain_ids, + columns_names=leaderboard.columns_names, ) except actions.LeaderboardUpdateError as e: logger.error(f"Error while updating leaderboard: {e}") @@ -233,6 +245,10 @@ async def update_leaderboard( title=updated_leaderboard.title, # type: ignore description=updated_leaderboard.description, # type: ignore resource_id=updated_leaderboard.resource_id, # type: ignore + public=updated_leaderboard.public, # type: ignore + show_connect=updated_leaderboard.show_connect, # type: ignore + blockchain_ids=updated_leaderboard.blockchain_ids, # type: ignore + columns_names=updated_leaderboard.columns_names, # type: ignore created_at=updated_leaderboard.created_at, # type: ignore updated_at=updated_leaderboard.updated_at, # type: ignore ) @@ -292,6 +308,11 @@ async def delete_leaderboard( id=deleted_leaderboard.id, # type: ignore title=deleted_leaderboard.title, # type: ignore description=deleted_leaderboard.description, # type: ignore + resource_id=deleted_leaderboard.resource_id, # type: ignore + public=deleted_leaderboard.public, # type: ignore + show_connect=deleted_leaderboard.show_connect, # type: ignore + blockchain_ids=deleted_leaderboard.blockchain_ids, # type: ignore + columns_names=deleted_leaderboard.columns_names, # type: ignore created_at=deleted_leaderboard.created_at, # type: ignore updated_at=deleted_leaderboard.updated_at, # type: ignore ) @@ -330,6 +351,10 @@ async def get_leaderboards( title=leaderboard.title, # type: ignore description=leaderboard.description, # type: ignore resource_id=leaderboard.resource_id, # type: ignore + public=leaderboard.public, # type: ignore + show_connect=leaderboard.show_connect, # type: ignore + blockchain_ids=leaderboard.blockchain_ids, # type: ignore + columns_names=leaderboard.columns_names, # type: ignore created_at=leaderboard.created_at, # type: ignore updated_at=leaderboard.updated_at, # type: ignore ) From 5878e66aaf57d02b942f8dbb3ec841e8c995a7b8 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 15 Nov 2023 17:28:15 +0200 Subject: [PATCH 02/35] Add fixes for get leaderboard. --- ...y => 71e888082a6d_leaderboard_metadata.py} | 14 ++-- engineapi/engineapi/actions.py | 12 ++-- engineapi/engineapi/data.py | 22 +++++-- engineapi/engineapi/models.py | 3 +- engineapi/engineapi/routes/leaderboard.py | 64 +++++++++++++------ 5 files changed, 76 insertions(+), 39 deletions(-) rename engineapi/alembic/versions/{e86bb1230bca_leaderboard_metadata.py => 71e888082a6d_leaderboard_metadata.py} (81%) diff --git a/engineapi/alembic/versions/e86bb1230bca_leaderboard_metadata.py b/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py similarity index 81% rename from engineapi/alembic/versions/e86bb1230bca_leaderboard_metadata.py rename to engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py index 3198bc75..da9fa9d1 100644 --- a/engineapi/alembic/versions/e86bb1230bca_leaderboard_metadata.py +++ b/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py @@ -1,8 +1,8 @@ """leaderboard metadata -Revision ID: e86bb1230bca +Revision ID: 71e888082a6d Revises: 040f2dfde5a5 -Create Date: 2023-11-09 16:43:21.553490 +Create Date: 2023-11-15 13:21:16.108399 """ from alembic import op @@ -10,7 +10,7 @@ import sqlalchemy as sa from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = "e86bb1230bca" +revision = "71e888082a6d" down_revision = "040f2dfde5a5" branch_labels = None depends_on = None @@ -29,11 +29,13 @@ def upgrade(): ) op.add_column( "leaderboards", - sa.Column("show_connect", sa.Boolean(), nullable=False, server_default="false"), + sa.Column( + "wallet_connect", sa.Boolean(), nullable=False, server_default="false" + ), ) op.add_column( "leaderboards", - sa.Column("public", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("public", sa.Boolean(), nullable=False, server_default="true"), ) op.add_column( "leaderboards", @@ -51,6 +53,6 @@ def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.drop_column("leaderboards", "columns_names") op.drop_column("leaderboards", "public") - op.drop_column("leaderboards", "show_connect") + op.drop_column("leaderboards", "wallet_connect") op.drop_column("leaderboards", "blockchain_ids") # ### end Alembic commands ### diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index feb5fe8d..05310ed7 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1257,7 +1257,7 @@ def create_leaderboard( description: Optional[str], token: Optional[Union[uuid.UUID, str]] = None, public: bool = False, - show_connect: bool = False, + wallet_connect: bool = False, blockchain_ids: Optional[List[int]] = None, columns_names: Optional[Dict[str, str]] = None, ) -> Leaderboard: @@ -1272,7 +1272,7 @@ def create_leaderboard( title=title, description=description, public=public, - show_connect=show_connect, + wallet_connect=wallet_connect, blockchain_ids=blockchain_ids, columns_names=columns_names, ) @@ -1335,7 +1335,7 @@ def update_leaderboard( title: Optional[str], description: Optional[str], public: Optional[bool], - show_connect: Optional[bool], + wallet_connect: Optional[bool], blockchain_ids: Optional[List[int]], columns_names: Optional[Dict[str, str]], ) -> Leaderboard: @@ -1353,8 +1353,8 @@ def update_leaderboard( leaderboard.description = description if public is not None: leaderboard.public = public - if show_connect is not None: - leaderboard.show_connect = show_connect + if wallet_connect is not None: + leaderboard.wallet_connect = wallet_connect if blockchain_ids is not None: leaderboard.blockchain_ids = blockchain_ids if columns_names is not None: @@ -1369,7 +1369,7 @@ def get_leaderboard_by_id(db_session: Session, leaderboard_id) -> Leaderboard: """ Get the leaderboard by id """ - return db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).filter(leaderboard.public == True).one() # type: ignore + return db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).filter(Leaderboard.public == True).one() # type: ignore def get_leaderboard_by_title(db_session: Session, title) -> Leaderboard: diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 49fa92be..9fffad0b 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -342,6 +342,16 @@ class Score(BaseModel): points_data: Dict[str, Any] +class LeaderboardUnformattedPosition(BaseModel): + column_1: str = Field(serialization_alias="address") + column_2: int = Field(serialization_alias="rank") + column_3: int = Field(serialization_alias="score") + column_4: Dict[str, Any] = Field(serialization_alias="points_data") + + class Config: + orm_mode = True + + class LeaderboardPosition(BaseModel): address: str rank: int @@ -371,7 +381,7 @@ class Leaderboard(BaseModel): description: Optional[str] = None resource_id: Optional[UUID] = None public: Optional[bool] = False - show_connect: Optional[bool] = False + wallet_connect: Optional[bool] = False blockchain_ids: Optional[List[int]] = None columns_names: Optional[Dict[str, str]] = None created_at: datetime @@ -390,7 +400,7 @@ class LeaderboardCreateRequest(BaseModel): title: str description: Optional[str] = None public: Optional[bool] = False - show_connect: Optional[bool] = False + wallet_connect: Optional[bool] = False blockchain_ids: Optional[List[int]] = None columns_names: Optional[Dict[str, str]] = None @@ -401,7 +411,7 @@ class LeaderboardCreatedResponse(BaseModel): description: Optional[str] = None resource_id: Optional[UUID] = None public: Optional[bool] = False - show_connect: Optional[bool] = False + wallet_connect: Optional[bool] = False blockchain_ids: Optional[List[int]] = None columns_names: Optional[Dict[str, str]] = None created_at: datetime @@ -417,7 +427,7 @@ class LeaderboardUpdatedResponse(BaseModel): description: Optional[str] = None resource_id: Optional[UUID] = None public: Optional[bool] = False - show_connect: Optional[bool] = False + wallet_connect: Optional[bool] = False blockchain_ids: Optional[List[int]] = None columns_names: Optional[Dict[str, str]] = None created_at: datetime @@ -431,7 +441,7 @@ class LeaderboardUpdateRequest(BaseModel): title: Optional[str] = None description: Optional[str] = None public: Optional[bool] = False - show_connect: Optional[bool] = False + wallet_connect: Optional[bool] = False blockchain_ids: Optional[List[int]] = None columns_names: Optional[Dict[str, str]] = None @@ -442,7 +452,7 @@ class LeaderboardDeletedResponse(BaseModel): description: Optional[str] = None resource_id: Optional[UUID] = None public: Optional[bool] = False - show_connect: Optional[bool] = False + wallet_connect: Optional[bool] = False blockchain_ids: Optional[List[int]] = None columns_names: Optional[Dict[str, str]] = None created_at: datetime diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index a937fe07..c786ea5c 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -348,7 +348,8 @@ class Leaderboard(Base): # type: ignore description = Column(String, nullable=True) resource_id = Column(UUID(as_uuid=True), nullable=True, index=True) blockchain_ids = Column(ARRAY(Integer), nullable=False, default=[]) - show_connect = Column(Boolean, default=False, nullable=False) + + wallet_connect = Column(Boolean, default=False, nullable=False) public = Column(Boolean, default=False, nullable=False) columns_names = Column(JSONB, nullable=False, default={}) created_at = Column( diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index fbbbbb08..6f79e3ce 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -2,7 +2,7 @@ Leaderboard API. """ import logging -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Any, Union from uuid import UUID from bugout.exceptions import BugoutResponseException @@ -87,21 +87,33 @@ app.add_middleware( ) -@app.get("", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"]) -@app.get("/", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"]) +@app.get( + "", + response_model=Union[ + List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition] + ], + tags=["Public Endpoints"], +) +@app.get( + "/", + response_model=Union[ + List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition] + ], + tags=["Public Endpoints"], +) async def leaderboard( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), limit: int = Query(10), offset: int = Query(0), db_session: Session = Depends(db.yield_db_session), -) -> List[data.LeaderboardPosition]: +) -> Any: """ Returns the leaderboard positions. """ ### Check if leaderboard exists try: - actions.get_leaderboard_by_id(db_session, leaderboard_id) + leaderboard = actions.get_leaderboard_by_id(db_session, leaderboard_id) except NoResultFound as e: raise EngineHTTPException( status_code=404, @@ -114,16 +126,28 @@ async def leaderboard( leaderboard_positions = actions.get_leaderboard_positions( db_session, leaderboard_id, limit, offset ) + if len(leaderboard.columns_names) > 0: + # breakpoint() + result = [ + data.LeaderboardUnformattedPosition( + column_1=position[1], + column_2=position[2], + column_3=position[4], + column_4=position[3], + ) + for position in leaderboard_positions + ] - result = [ - data.LeaderboardPosition( - address=position.address, - score=position.score, - rank=position.rank, - points_data=position.points_data, - ) - for position in leaderboard_positions - ] + else: + result = [ + data.LeaderboardPosition( + address=position.address, + score=position.score, + rank=position.rank, + points_data=position.points_data, + ) + for position in leaderboard_positions + ] return result @@ -153,7 +177,7 @@ async def create_leaderboard( description=leaderboard.description, token=token, public=leaderboard.public, - show_connect=leaderboard.show_connect, + wallet_connect=leaderboard.wallet_connect, blockchain_ids=leaderboard.blockchain_ids, columns_names=leaderboard.columns_names, ) @@ -176,7 +200,7 @@ async def create_leaderboard( description=created_leaderboard.description, # type: ignore resource_id=created_leaderboard.resource_id, # type: ignore public=created_leaderboard.public, # type: ignore - show_connect=created_leaderboard.show_connect, # type: ignore + wallet_connect=created_leaderboard.wallet_connect, # type: ignore blockchain_ids=created_leaderboard.blockchain_ids, # type: ignore columns_names=created_leaderboard.columns_names, # type: ignore created_at=created_leaderboard.created_at, # type: ignore @@ -225,7 +249,7 @@ async def update_leaderboard( title=leaderboard.title, description=leaderboard.description, public=leaderboard.public, - show_connect=leaderboard.show_connect, + wallet_connect=leaderboard.wallet_connect, blockchain_ids=leaderboard.blockchain_ids, columns_names=leaderboard.columns_names, ) @@ -246,7 +270,7 @@ async def update_leaderboard( description=updated_leaderboard.description, # type: ignore resource_id=updated_leaderboard.resource_id, # type: ignore public=updated_leaderboard.public, # type: ignore - show_connect=updated_leaderboard.show_connect, # type: ignore + wallet_connect=updated_leaderboard.wallet_connect, # type: ignore blockchain_ids=updated_leaderboard.blockchain_ids, # type: ignore columns_names=updated_leaderboard.columns_names, # type: ignore created_at=updated_leaderboard.created_at, # type: ignore @@ -310,7 +334,7 @@ async def delete_leaderboard( description=deleted_leaderboard.description, # type: ignore resource_id=deleted_leaderboard.resource_id, # type: ignore public=deleted_leaderboard.public, # type: ignore - show_connect=deleted_leaderboard.show_connect, # type: ignore + wallet_connect=deleted_leaderboard.wallet_connect, # type: ignore blockchain_ids=deleted_leaderboard.blockchain_ids, # type: ignore columns_names=deleted_leaderboard.columns_names, # type: ignore created_at=deleted_leaderboard.created_at, # type: ignore @@ -352,7 +376,7 @@ async def get_leaderboards( description=leaderboard.description, # type: ignore resource_id=leaderboard.resource_id, # type: ignore public=leaderboard.public, # type: ignore - show_connect=leaderboard.show_connect, # type: ignore + wallet_connect=leaderboard.wallet_connect, # type: ignore blockchain_ids=leaderboard.blockchain_ids, # type: ignore columns_names=leaderboard.columns_names, # type: ignore created_at=leaderboard.created_at, # type: ignore From 324ff584662bf732ecaa3d90f122259413d532cb Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 15 Nov 2023 18:00:18 +0200 Subject: [PATCH 03/35] Add columns names default. --- engineapi/engineapi/actions.py | 36 ++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 05310ed7..69222078 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -16,7 +16,14 @@ from sqlalchemy.engine import Row from web3 import Web3 from web3.types import ChecksumAddress -from .data import Score, LeaderboardScore, LeaderboardConfigUpdate, LeaderboardConfig +from .data import ( + Score, + LeaderboardScore, + LeaderboardConfigUpdate, + LeaderboardConfig, + LeaderboardPosition, + LeaderboardUnformattedPosition, +) from .contracts import Dropper_interface, ERC20_interface, Terminus_interface from .models import ( DropperClaimant, @@ -96,6 +103,27 @@ BATCH_SIGNATURE_PAGE_SIZE = 500 logger = logging.getLogger(__name__) +def get_default_columns_names(columns_names: Optional[Dict[str, str]] = {}): + default_columns_names = {} + + # Retrieve ordered list of field names from LeaderboardPosition + ordered_fields = list(LeaderboardPosition.__annotations__.keys()) + + # Mapping these to the corresponding columns in LeaderboardUnformattedPosition + for index, field_name in enumerate(ordered_fields, start=1): + # Construct the column name like "column_1", "column_2", etc. + column_name = f"column_{index}" + + # If custom column name is provided in columns_names, use it + if columns_names and field_name in columns_names: + default_columns_names[column_name] = columns_names[field_name] + else: + # Otherwise, use the default field name in title case + default_columns_names[column_name] = field_name.replace("_", " ").title() + + return default_columns_names + + def create_dropper_contract( db_session: Session, blockchain: Optional[str], @@ -1259,12 +1287,15 @@ def create_leaderboard( public: bool = False, wallet_connect: bool = False, blockchain_ids: Optional[List[int]] = None, - columns_names: Optional[Dict[str, str]] = None, + columns_names: Optional[Dict[str, str]] = {}, ) -> Leaderboard: """ Create a leaderboard """ + if len(columns_names) > 0: + columns_names = get_default_columns_names(columns_names) + if not token: token = uuid.UUID(MOONSTREAM_ADMIN_ACCESS_TOKEN) try: @@ -1358,6 +1389,7 @@ def update_leaderboard( if blockchain_ids is not None: leaderboard.blockchain_ids = blockchain_ids if columns_names is not None: + columns_names = get_default_columns_names(columns_names) leaderboard.columns_names = columns_names db_session.commit() From 77dfd5d070c3c5dd1c1e839040b7b77dae253ddf Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 16 Nov 2023 06:50:56 +0200 Subject: [PATCH 04/35] Update Quartiles / Position / Rank / Ranks endpoints. --- engineapi/engineapi/actions.py | 4 +- engineapi/engineapi/routes/leaderboard.py | 113 ++++++++++++++++------ 2 files changed, 83 insertions(+), 34 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 69222078..d74e8c58 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1193,8 +1193,8 @@ def get_qurtiles( """ query = db_session.query( - LeaderboardScores.address, - LeaderboardScores.score, + LeaderboardScores.address.label("address"), + LeaderboardScores.score.label("score"), func.rank().over(order_by=LeaderboardScores.score.desc()).label("rank"), ).filter(LeaderboardScores.leaderboard_id == leaderboard_id) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 6f79e3ce..5af65f9a 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -127,7 +127,6 @@ async def leaderboard( db_session, leaderboard_id, limit, offset ) if len(leaderboard.columns_names) > 0: - # breakpoint() result = [ data.LeaderboardUnformattedPosition( column_1=position[1], @@ -489,7 +488,7 @@ async def quartiles( """ ### Check if leaderboard exists try: - actions.get_leaderboard_by_id(db_session, leaderboard_id) + leaderboard = actions.get_leaderboard_by_id(db_session, leaderboard_id) except NoResultFound as e: raise EngineHTTPException( status_code=404, @@ -501,23 +500,45 @@ async def quartiles( try: q1, q2, q3 = actions.get_qurtiles(db_session, leaderboard_id) - except actions.LeaderboardIsEmpty: raise EngineHTTPException(status_code=204, detail="Leaderboard is empty.") except Exception as e: logger.error(f"Error while getting quartiles: {e}") raise EngineHTTPException(status_code=500, detail="Internal server error") - return data.QuartilesResponse( - percentile_25={"address": q1[0], "score": q1[1], "rank": q1[2]}, - percentile_50={"address": q2[0], "score": q2[1], "rank": q2[2]}, - percentile_75={"address": q3[0], "score": q3[1], "rank": q3[2]}, - ) + if len(leaderboard.columns_names) > 0: + result = data.QuartilesResponse( + percentile_25={ + "column_1": q1.address, + "column_2": q1.rank, + "column_3": q1.score, + }, + percentile_50={ + "column_1": q2.address, + "column_2": q2.rank, + "column_3": q2.score, + }, + percentile_75={ + "column_1": q3.address, + "column_2": q3.rank, + "column_3": q3.score, + }, + ) + else: + result = data.QuartilesResponse( + percentile_25={"address": q1.address, "rank": q1.rank, "score": q1.score}, + percentile_50={"address": q2.address, "rank": q2.rank, "score": q2.score}, + percentile_75={"address": q3.address, "rank": q3.rank, "score": q3.score}, + ) + + return result @app.get( "/position", - response_model=List[data.LeaderboardPosition], + response_model=Union[ + List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition] + ], tags=["Public Endpoints"], ) async def position( @@ -530,7 +551,7 @@ async def position( True, description="Normalize addresses to checksum." ), db_session: Session = Depends(db.yield_db_session), -) -> List[data.LeaderboardPosition]: +) -> Union[List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition]]: """ Returns the leaderboard posotion for the given address. With given window size. @@ -538,7 +559,7 @@ async def position( ### Check if leaderboard exists try: - actions.get_leaderboard_by_id(db_session, leaderboard_id) + leaderboard = actions.get_leaderboard_by_id(db_session, leaderboard_id) except NoResultFound as e: raise EngineHTTPException( status_code=404, @@ -555,21 +576,37 @@ async def position( db_session, leaderboard_id, address, window_size, limit, offset ) - results = [ - data.LeaderboardPosition( - address=position.address, - score=position.score, - rank=position.rank, - points_data=position.points_data, - ) - for position in positions - ] + if len(leaderboard.columns_names) > 0: + results = [ + data.LeaderboardUnformattedPosition( + column_1=position.address, + column_2=position.rank, + column_3=position.score, + column_4=position.points_data, + ) + for position in positions + ] + + else: + results = [ + data.LeaderboardPosition( + address=position.address, + score=position.score, + rank=position.rank, + points_data=position.points_data, + ) + for position in positions + ] return results @app.get( - "/rank", response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"] + "/rank", + response_model=Union[ + List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition] + ], + tags=["Public Endpoints"], ) async def rank( leaderboard_id: UUID = Query(..., description="Leaderboard ID"), @@ -577,14 +614,14 @@ async def rank( limit: Optional[int] = Query(None), offset: Optional[int] = Query(None), db_session: Session = Depends(db.yield_db_session), -) -> List[data.LeaderboardPosition]: +) -> Union[List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition]]: """ Returns the leaderboard scores for the given rank. """ ### Check if leaderboard exists try: - actions.get_leaderboard_by_id(db_session, leaderboard_id) + leaderboard = actions.get_leaderboard_by_id(db_session, leaderboard_id) except NoResultFound as e: raise EngineHTTPException( status_code=404, @@ -597,15 +634,27 @@ async def rank( leaderboard_rank = actions.get_rank( db_session, leaderboard_id, rank, limit=limit, offset=offset ) - results = [ - data.LeaderboardPosition( - address=rank_position.address, - score=rank_position.score, - rank=rank_position.rank, - points_data=rank_position.points_data, - ) - for rank_position in leaderboard_rank - ] + + if len(leaderboard.columns_names) > 0: + results = [ + data.LeaderboardUnformattedPosition( + column_1=position.address, + column_2=position.rank, + column_3=position.score, + column_4=position.points_data, + ) + for position in leaderboard_rank + ] + else: + results = [ + data.LeaderboardPosition( + address=position.address, + score=position.score, + rank=position.rank, + points_data=position.points_data, + ) + for position in leaderboard_rank + ] return results From e9273e2d383c3cdc24665a929abed1f018bf3fad Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 21 Nov 2023 08:03:34 +0200 Subject: [PATCH 05/35] Remove public column and add suggested changes. --- .../71e888082a6d_leaderboard_metadata.py | 5 --- engineapi/engineapi/actions.py | 15 +++---- engineapi/engineapi/data.py | 42 ++++++++----------- engineapi/engineapi/models.py | 2 - engineapi/engineapi/routes/leaderboard.py | 6 --- 5 files changed, 26 insertions(+), 44 deletions(-) diff --git a/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py b/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py index da9fa9d1..43ef9d0c 100644 --- a/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py +++ b/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py @@ -33,10 +33,6 @@ def upgrade(): "wallet_connect", sa.Boolean(), nullable=False, server_default="false" ), ) - op.add_column( - "leaderboards", - sa.Column("public", sa.Boolean(), nullable=False, server_default="true"), - ) op.add_column( "leaderboards", sa.Column( @@ -52,7 +48,6 @@ def upgrade(): def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.drop_column("leaderboards", "columns_names") - op.drop_column("leaderboards", "public") op.drop_column("leaderboards", "wallet_connect") op.drop_column("leaderboards", "blockchain_ids") # ### end Alembic commands ### diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index d74e8c58..1d1cd12d 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1284,9 +1284,8 @@ def create_leaderboard( title: str, description: Optional[str], token: Optional[Union[uuid.UUID, str]] = None, - public: bool = False, wallet_connect: bool = False, - blockchain_ids: Optional[List[int]] = None, + blockchain_ids: List[int] = [], columns_names: Optional[Dict[str, str]] = {}, ) -> Leaderboard: """ @@ -1299,10 +1298,12 @@ def create_leaderboard( if not token: token = uuid.UUID(MOONSTREAM_ADMIN_ACCESS_TOKEN) try: + # deduplicate and sort + blockchain_ids = sorted(list(set(blockchain_ids))) + leaderboard = Leaderboard( title=title, description=description, - public=public, wallet_connect=wallet_connect, blockchain_ids=blockchain_ids, columns_names=columns_names, @@ -1365,7 +1366,6 @@ def update_leaderboard( leaderboard_id: uuid.UUID, title: Optional[str], description: Optional[str], - public: Optional[bool], wallet_connect: Optional[bool], blockchain_ids: Optional[List[int]], columns_names: Optional[Dict[str, str]], @@ -1382,12 +1382,13 @@ def update_leaderboard( leaderboard.title = title if description is not None: leaderboard.description = description - if public is not None: - leaderboard.public = public if wallet_connect is not None: leaderboard.wallet_connect = wallet_connect if blockchain_ids is not None: + # deduplicate and sort + blockchain_ids = sorted(list(set(blockchain_ids))) leaderboard.blockchain_ids = blockchain_ids + if columns_names is not None: columns_names = get_default_columns_names(columns_names) leaderboard.columns_names = columns_names @@ -1401,7 +1402,7 @@ def get_leaderboard_by_id(db_session: Session, leaderboard_id) -> Leaderboard: """ Get the leaderboard by id """ - return db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).filter(Leaderboard.public == True).one() # type: ignore + return db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() # type: ignore def get_leaderboard_by_title(db_session: Session, title) -> Leaderboard: diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 9fffad0b..4ce5f07c 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -380,10 +380,9 @@ class Leaderboard(BaseModel): title: str description: Optional[str] = None resource_id: Optional[UUID] = None - public: Optional[bool] = False - wallet_connect: Optional[bool] = False - blockchain_ids: Optional[List[int]] = None - columns_names: Optional[Dict[str, str]] = None + wallet_connect: bool = False + blockchain_ids: List[int] = Field(default_factory=list) + columns_names: Dict[str, str] = Field(default_factory=dict) created_at: datetime updated_at: datetime @@ -399,10 +398,9 @@ class LeaderboardInfoResponse(BaseModel): class LeaderboardCreateRequest(BaseModel): title: str description: Optional[str] = None - public: Optional[bool] = False - wallet_connect: Optional[bool] = False - blockchain_ids: Optional[List[int]] = None - columns_names: Optional[Dict[str, str]] = None + wallet_connect: bool = False + blockchain_ids: List[int] = Field(default_factory=list) + columns_names: Dict[str, str] = Field(default_factory=dict) class LeaderboardCreatedResponse(BaseModel): @@ -410,10 +408,9 @@ class LeaderboardCreatedResponse(BaseModel): title: str description: Optional[str] = None resource_id: Optional[UUID] = None - public: Optional[bool] = False - wallet_connect: Optional[bool] = False - blockchain_ids: Optional[List[int]] = None - columns_names: Optional[Dict[str, str]] = None + wallet_connect: bool = False + blockchain_ids: List[int] = Field(default_factory=list) + columns_names: Dict[str, str] = Field(default_factory=dict) created_at: datetime updated_at: datetime @@ -426,10 +423,9 @@ class LeaderboardUpdatedResponse(BaseModel): title: str description: Optional[str] = None resource_id: Optional[UUID] = None - public: Optional[bool] = False - wallet_connect: Optional[bool] = False - blockchain_ids: Optional[List[int]] = None - columns_names: Optional[Dict[str, str]] = None + wallet_connect: bool = False + blockchain_ids: List[int] = Field(default_factory=list) + columns_names: Dict[str, str] = Field(default_factory=dict) created_at: datetime updated_at: datetime @@ -440,10 +436,9 @@ class LeaderboardUpdatedResponse(BaseModel): class LeaderboardUpdateRequest(BaseModel): title: Optional[str] = None description: Optional[str] = None - public: Optional[bool] = False - wallet_connect: Optional[bool] = False - blockchain_ids: Optional[List[int]] = None - columns_names: Optional[Dict[str, str]] = None + wallet_connect: bool = False + blockchain_ids: List[int] = Field(default_factory=list) + columns_names: Dict[str, str] = Field(default_factory=dict) class LeaderboardDeletedResponse(BaseModel): @@ -451,10 +446,9 @@ class LeaderboardDeletedResponse(BaseModel): title: str description: Optional[str] = None resource_id: Optional[UUID] = None - public: Optional[bool] = False - wallet_connect: Optional[bool] = False - blockchain_ids: Optional[List[int]] = None - columns_names: Optional[Dict[str, str]] = None + wallet_connect: bool = False + blockchain_ids: List[int] = Field(default_factory=list) + columns_names: Dict[str, str] = Field(default_factory=dict) created_at: datetime updated_at: datetime diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index c786ea5c..b5651247 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -335,7 +335,6 @@ class CallRequest(Base): class Leaderboard(Base): # type: ignore __tablename__ = "leaderboards" - # __table_args__ = (UniqueConstraint("dropper_contract_id", "address"),) id = Column( UUID(as_uuid=True), @@ -350,7 +349,6 @@ class Leaderboard(Base): # type: ignore blockchain_ids = Column(ARRAY(Integer), nullable=False, default=[]) wallet_connect = Column(Boolean, default=False, nullable=False) - public = Column(Boolean, default=False, nullable=False) columns_names = Column(JSONB, nullable=False, default={}) created_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 5af65f9a..d1b46bea 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -175,7 +175,6 @@ async def create_leaderboard( title=leaderboard.title, description=leaderboard.description, token=token, - public=leaderboard.public, wallet_connect=leaderboard.wallet_connect, blockchain_ids=leaderboard.blockchain_ids, columns_names=leaderboard.columns_names, @@ -198,7 +197,6 @@ async def create_leaderboard( title=created_leaderboard.title, # type: ignore description=created_leaderboard.description, # type: ignore resource_id=created_leaderboard.resource_id, # type: ignore - public=created_leaderboard.public, # type: ignore wallet_connect=created_leaderboard.wallet_connect, # type: ignore blockchain_ids=created_leaderboard.blockchain_ids, # type: ignore columns_names=created_leaderboard.columns_names, # type: ignore @@ -247,7 +245,6 @@ async def update_leaderboard( leaderboard_id=leaderboard_id, title=leaderboard.title, description=leaderboard.description, - public=leaderboard.public, wallet_connect=leaderboard.wallet_connect, blockchain_ids=leaderboard.blockchain_ids, columns_names=leaderboard.columns_names, @@ -268,7 +265,6 @@ async def update_leaderboard( title=updated_leaderboard.title, # type: ignore description=updated_leaderboard.description, # type: ignore resource_id=updated_leaderboard.resource_id, # type: ignore - public=updated_leaderboard.public, # type: ignore wallet_connect=updated_leaderboard.wallet_connect, # type: ignore blockchain_ids=updated_leaderboard.blockchain_ids, # type: ignore columns_names=updated_leaderboard.columns_names, # type: ignore @@ -332,7 +328,6 @@ async def delete_leaderboard( title=deleted_leaderboard.title, # type: ignore description=deleted_leaderboard.description, # type: ignore resource_id=deleted_leaderboard.resource_id, # type: ignore - public=deleted_leaderboard.public, # type: ignore wallet_connect=deleted_leaderboard.wallet_connect, # type: ignore blockchain_ids=deleted_leaderboard.blockchain_ids, # type: ignore columns_names=deleted_leaderboard.columns_names, # type: ignore @@ -374,7 +369,6 @@ async def get_leaderboards( title=leaderboard.title, # type: ignore description=leaderboard.description, # type: ignore resource_id=leaderboard.resource_id, # type: ignore - public=leaderboard.public, # type: ignore wallet_connect=leaderboard.wallet_connect, # type: ignore blockchain_ids=leaderboard.blockchain_ids, # type: ignore columns_names=leaderboard.columns_names, # type: ignore From 6554b1632dfb72e9c338aa3e15121608337c5807 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 05:24:59 +0200 Subject: [PATCH 06/35] Add WETH and MUTE. --- crawlers/mooncrawl/mooncrawl/settings.py | 1 + .../state_crawler/jobs/zksync-era-jobs.json | 75 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 79a3a8a5..fe14e66c 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -241,6 +241,7 @@ multicall_contracts: Dict[AvailableBlockchainType, str] = { AvailableBlockchainType.POLYGON: "0xc8E51042792d7405184DfCa245F2d27B94D013b6", AvailableBlockchainType.MUMBAI: "0xe9939e7Ea7D7fb619Ac57f648Da7B1D425832631", AvailableBlockchainType.ETHEREUM: "0x5BA1e12693Dc8F9c48aAD8770482f4739bEeD696", + AvailableBlockchainType.ZKSYNC_ERA: "0xF9cda624FBC7e059355ce98a31693d299FACd963", } diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json new file mode 100644 index 00000000..6301e903 --- /dev/null +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json @@ -0,0 +1,75 @@ +[ + { + "constant": true, + "inputs": [], + "name": "getReserves", + "outputs": [ + { + "internalType": "uint", + "name": "", + "type": "uint256" + }, + { + "internalType": "uint", + "name": "", + "type": "uint256" + } + ], + "payable": false, + "address": "0x80115c708E12eDd42E504c1cD52Aea96C547c05c", + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "getReserves", + "outputs": [ + { + "internalType": "uint112", + "name": "_reserve0", + "type": "uint112" + }, + { + "internalType": "uint112", + "name": "_reserve1", + "type": "uint112" + }, + { + "internalType": "uint32", + "name": "_blockTimestampLast", + "type": "uint32" + } + ], + "payable": false, + "address": "0xb85feb6aF3412d690DFDA280b73EaED73a2315bC", + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "getReserves", + "outputs": [ + { + "internalType": "uint112", + "name": "_reserve0", + "type": "uint112" + }, + { + "internalType": "uint112", + "name": "_reserve1", + "type": "uint112" + }, + { + "internalType": "uint32", + "name": "_blockTimestampLast", + "type": "uint32" + } + ], + "payable": false, + "address": "0xDFAaB828f5F515E104BaaBa4d8D554DA9096f0e4", + "stateMutability": "view", + "type": "function" + } +] \ No newline at end of file From 21fb0ca0d7c94a29eeef335967739d7ab2424aed Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 06:42:36 +0200 Subject: [PATCH 07/35] Add batching leaderboard generator. --- .../mooncrawl/leaderboards_generator/cli.py | 66 ++++++--- .../mooncrawl/leaderboards_generator/utils.py | 126 +++++++++++++++++- crawlers/mooncrawl/mooncrawl/settings.py | 4 + 3 files changed, 174 insertions(+), 22 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py index 39b8f0f7..7e382109 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -7,12 +7,14 @@ import uuid import requests # type: ignore from bugout.data import BugoutSearchResult -from .utils import get_results_for_moonstream_query +from .utils import get_results_for_moonstream_query, leaderboard_push_batch from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, MOONSTREAM_API_URL, MOONSTREAM_ENGINE_URL, + MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE, + MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS, ) from ..settings import bugout_client as bc @@ -35,10 +37,15 @@ def handle_leaderboards(args: argparse.Namespace) -> None: ### get leaderboard journal + leaderboard_push_batch_size = args.leaderboard_push_batch_size + + leaderboard_push_timeout_seconds = args.leaderboard_push_timeout_seconds + query = "#leaderboard #status:active" - if args.leaderboard_id: # way to run only one leaderboard - query += f" #leaderboard_id:{args.leaderboard_id}" + if args.leaderboard_id: # way to run only one leaderboard without status:active + query = f"#leaderboard #leaderboard_id:{args.leaderboard_id}" + try: leaderboards = bc.search( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, @@ -116,26 +123,33 @@ def handle_leaderboards(args: argparse.Namespace) -> None: "Content-Type": "application/json", } - try: - leaderboard_api_response = requests.put( - leaderboard_push_api_url, - json=query_results["data"], - headers=leaderboard_api_headers, - timeout=10, + if len(query_results["data"]) > leaderboard_push_batch_size: + logger.info( + f"Pushing {len(query_results['data'])} scores to leaderboard {leaderboard_id} in batches of {leaderboard_push_batch_size}" ) - except Exception as e: - logger.error( - f"Could not push results to leaderboard API: {e} for leaderboard {leaderboard_id}" + leaderboard_push_batch( + leaderboard_id, + leaderboard_data, + query_results["data"], + leaderboard_api_headers, + leaderboard_push_batch_size, + timeout=leaderboard_push_timeout_seconds, ) - continue - try: - leaderboard_api_response.raise_for_status() - except requests.exceptions.HTTPError as http_error: - logger.error( - f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" - ) - continue + else: + try: + leaderboard_api_response = requests.put( + leaderboard_push_api_url, + json=query_results["data"], + headers=leaderboard_api_headers, + timeout=leaderboard_push_timeout_seconds, + ) + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" + ) + continue ### get leaderboard from leaderboard API @@ -213,6 +227,18 @@ def main(): required=True, help="Moonstream Access Token to use for Moonstream Query API requests", ) + leaderboard_generator_parser.add_argument( + "--leaderboard-push-batch-size", + type=int, + default=MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE, + help="Number of scores to push to leaderboard API at once", + ) + leaderboard_generator_parser.add_argument( + "--leaderboard-push-timeout-seconds", + type=int, + default=MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS, + help="Timeout for leaderboard API requests", + ) leaderboard_generator_parser.set_defaults(func=handle_leaderboards) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py index fc4d1e01..1e08f668 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -3,12 +3,17 @@ import json import logging import os import time -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, List import requests # type: ignore -from ..settings import MOONSTREAM_API_URL +from ..settings import ( + MOONSTREAM_API_URL, + MOONSTREAM_ENGINE_URL, + MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE, + MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS, +) logging.basicConfig() @@ -101,3 +106,120 @@ def get_results_for_moonstream_query( keep_going = num_retries <= max_retries return result + + +def get_data_from_url(url): + response = requests.get(url) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to get data: HTTP {response.status_code}") + + +def chunk_data(data, chunk_size=100000): + for i in range(0, len(data), chunk_size): + yield data[i : i + chunk_size] + + +def send_data_to_endpoint(chunks, endpoint_url, headers, timeout=10): + for index, chunk in enumerate(chunks): + try: + logger.info(f"Pushing chunk {index} to leaderboard API") + response = requests.put( + endpoint_url, headers=headers, json=chunk, timeout=timeout + ) + + response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" + ) + continue + + +def leaderboard_push_batch( + leaderboard_id: str, + leaderboard_config: Dict[str, Any], + data: List[Dict[str, Any]], + headers: Dict[str, str], + batch_size: int = MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE, + timeout: int = 10, +) -> None: + """ + Push leaderboard data to the leaderboard API in batches. + """ + + ## first step create leaderboard version + + leaderboard_version_api_url = ( + f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/versions" + ) + + json_data = { + "publish": False, + } + + leaderboard_api_response = requests.post( + leaderboard_version_api_url, json=json_data, headers=headers, timeout=5 + ) + + try: + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not create leaderboard version: {http_error.response.text} with status code {http_error.response.status_code}" + ) + return + + leaderboard_version_id = leaderboard_api_response.json()["version"] + + ## second step push data to leaderboard version + + leaderboard_version_push_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/versions/{leaderboard_version_id}/scores?normalize_addresses={leaderboard_config['normalize_addresses']}&overwrite=false" + + chunks = chunk_data(data, chunk_size=batch_size) + + send_data_to_endpoint( + chunks, leaderboard_version_push_api_url, headers, timeout=timeout + ) + + ## third step publish leaderboard version + + leaderboard_version_publish_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/versions/{leaderboard_version_id}" + + json_data = { + "publish": True, + } + + try: + leaderboard_api_response = requests.put( + leaderboard_version_publish_api_url, + json=json_data, + headers=headers, + timeout=5, + ) + + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not publish leaderboard version: {http_error.response.text} with status code {http_error.response.status_code}" + ) + return + + ## delete leaderboard version -1 + + try: + leaderboard_version_delete_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/versions/{leaderboard_version_id - 1}" + + leaderboard_api_response = requests.delete( + leaderboard_version_delete_api_url, + headers=headers, + timeout=5, + ) + + leaderboard_api_response.raise_for_status() + except requests.exceptions.HTTPError as http_error: + logger.error( + f"Could not delete leaderboard version: {http_error.response.text} with status code {http_error.response.status_code}" + ) + return diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 79a3a8a5..2faafec2 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -321,3 +321,7 @@ if MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID == "": raise ValueError( "MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID environment variable must be set" ) + + +MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE = 20000 +MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS = 60 From 5a72358e994c8f60b7ce230724ad5ab5cb503e0b Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 10:41:51 +0200 Subject: [PATCH 08/35] Add deploy files. --- crawlers/deploy/deploy.bash | 21 +++++++++++++++++++ .../deploy/zksync-era-state-clean.service | 11 ++++++++++ crawlers/deploy/zksync-era-state-clean.timer | 9 ++++++++ crawlers/deploy/zksync-era-state.service | 11 ++++++++++ crawlers/deploy/zksync-era-state.timer | 9 ++++++++ 5 files changed, 61 insertions(+) create mode 100644 crawlers/deploy/zksync-era-state-clean.service create mode 100644 crawlers/deploy/zksync-era-state-clean.timer create mode 100644 crawlers/deploy/zksync-era-state.service create mode 100644 crawlers/deploy/zksync-era-state.timer diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 99e8b889..269a7cc8 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -115,6 +115,10 @@ ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="zksync-era-historical-cra ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="zksync-era-historical-crawl-transactions.timer" ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-historical-crawl-events.service" ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-historical-crawl-events.timer" +ZKSYNC_ERA_STATE_SERVICE_FILE="zksync-era-state.service" +ZKSYNC_ERA_STATE_TIMER_FILE="zksync-era-state.timer" +ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE="zksync-era-state-clean.service" +ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE="zksync-era-state-clean.timer" # ZkSync Era testnet ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service" @@ -550,6 +554,23 @@ cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubunt XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era state service and timer with: ${ZKSYNC_ERA_STATE_SERVICE_FILE}, ${ZKSYNC_ERA_STATE_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_STATE_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era state clean service and timer with: ${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}, ${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE}" # ZkSync Era testnet echo diff --git a/crawlers/deploy/zksync-era-state-clean.service b/crawlers/deploy/zksync-era-state-clean.service new file mode 100644 index 00000000..6b650d64 --- /dev/null +++ b/crawlers/deploy/zksync-era-state-clean.service @@ -0,0 +1,11 @@ +[Unit] +Description=Execute state clean labels crawler +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain zksync-era -N 10000 +CPUWeight=60 +SyslogIdentifier=zksync-era-state-clean \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-state-clean.timer b/crawlers/deploy/zksync-era-state-clean.timer new file mode 100644 index 00000000..c6e6ae99 --- /dev/null +++ b/crawlers/deploy/zksync-era-state-clean.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Execute Zksync Era state clean labels crawler each 25m + +[Timer] +OnBootSec=50s +OnUnitActiveSec=25m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/zksync-era-state.service b/crawlers/deploy/zksync-era-state.service new file mode 100644 index 00000000..8da61145 --- /dev/null +++ b/crawlers/deploy/zksync-era-state.service @@ -0,0 +1,11 @@ +[Unit] +Description=Execute state crawler +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +CPUWeight=60 +SyslogIdentifier=zksync-era-state \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-state.timer b/crawlers/deploy/zksync-era-state.timer new file mode 100644 index 00000000..b67cc1fe --- /dev/null +++ b/crawlers/deploy/zksync-era-state.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Execute Zksync Era state crawler each 10m + +[Timer] +OnBootSec=15s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target From f04ac73865e8fca3704f8d42f604aa5ef4afaebe Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 11:38:50 +0200 Subject: [PATCH 09/35] Fix blockchain in deploy script. --- crawlers/deploy/zksync-era-state.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/deploy/zksync-era-state.service b/crawlers/deploy/zksync-era-state.service index 8da61145..91ab6f4b 100644 --- a/crawlers/deploy/zksync-era-state.service +++ b/crawlers/deploy/zksync-era-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync-era --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json CPUWeight=60 SyslogIdentifier=zksync-era-state \ No newline at end of file From a8a0b165a226c0295381f0f0057aac4148b975af Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 11:47:09 +0200 Subject: [PATCH 10/35] Add fix. --- crawlers/deploy/zksync-era-state-clean.service | 2 +- crawlers/deploy/zksync-era-state.service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crawlers/deploy/zksync-era-state-clean.service b/crawlers/deploy/zksync-era-state-clean.service index 6b650d64..4cd68f25 100644 --- a/crawlers/deploy/zksync-era-state-clean.service +++ b/crawlers/deploy/zksync-era-state-clean.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain zksync-era -N 10000 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain zksync_era -N 10000 CPUWeight=60 SyslogIdentifier=zksync-era-state-clean \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-state.service b/crawlers/deploy/zksync-era-state.service index 91ab6f4b..573ae5d7 100644 --- a/crawlers/deploy/zksync-era-state.service +++ b/crawlers/deploy/zksync-era-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync-era --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json CPUWeight=60 SyslogIdentifier=zksync-era-state \ No newline at end of file From 691bdbf50fe1e60c65953a2368b7892ea1cbae0d Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 9 Dec 2023 12:17:32 +0200 Subject: [PATCH 11/35] Remove infura for zksync. --- crawlers/deploy/zksync-era-state.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/deploy/zksync-era-state.service b/crawlers/deploy/zksync-era-state.service index 573ae5d7..e92a6ccc 100644 --- a/crawlers/deploy/zksync-era-state.service +++ b/crawlers/deploy/zksync-era-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json CPUWeight=60 SyslogIdentifier=zksync-era-state \ No newline at end of file From 8a2e624e097a002a9b0b1b92de50a9032ddcb546 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 13 Dec 2023 21:39:52 +0200 Subject: [PATCH 12/35] Add changes. --- .../mooncrawl/leaderboards_generator/utils.py | 15 +++++---------- crawlers/mooncrawl/mooncrawl/settings.py | 2 +- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py index 1e08f668..98135370 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -116,11 +116,6 @@ def get_data_from_url(url): raise Exception(f"Failed to get data: HTTP {response.status_code}") -def chunk_data(data, chunk_size=100000): - for i in range(0, len(data), chunk_size): - yield data[i : i + chunk_size] - - def send_data_to_endpoint(chunks, endpoint_url, headers, timeout=10): for index, chunk in enumerate(chunks): try: @@ -134,7 +129,7 @@ def send_data_to_endpoint(chunks, endpoint_url, headers, timeout=10): logger.error( f"Could not push results to leaderboard API: {http_error.response.text} with status code {http_error.response.status_code}" ) - continue + raise http_error def leaderboard_push_batch( @@ -160,7 +155,7 @@ def leaderboard_push_batch( } leaderboard_api_response = requests.post( - leaderboard_version_api_url, json=json_data, headers=headers, timeout=5 + leaderboard_version_api_url, json=json_data, headers=headers, timeout=10 ) try: @@ -177,7 +172,7 @@ def leaderboard_push_batch( leaderboard_version_push_api_url = f"{MOONSTREAM_ENGINE_URL}/leaderboard/{leaderboard_id}/versions/{leaderboard_version_id}/scores?normalize_addresses={leaderboard_config['normalize_addresses']}&overwrite=false" - chunks = chunk_data(data, chunk_size=batch_size) + chunks = [data[x : x + batch_size] for x in range(0, len(data), batch_size)] send_data_to_endpoint( chunks, leaderboard_version_push_api_url, headers, timeout=timeout @@ -196,7 +191,7 @@ def leaderboard_push_batch( leaderboard_version_publish_api_url, json=json_data, headers=headers, - timeout=5, + timeout=10, ) leaderboard_api_response.raise_for_status() @@ -214,7 +209,7 @@ def leaderboard_push_batch( leaderboard_api_response = requests.delete( leaderboard_version_delete_api_url, headers=headers, - timeout=5, + timeout=timeout, ) leaderboard_api_response.raise_for_status() diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 2faafec2..ff843373 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -323,5 +323,5 @@ if MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID == "": ) -MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE = 20000 +MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE = 12000 MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS = 60 From 859f2ef2785f695eb5e3ab5a240e9139ddfbb124 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 13 Dec 2023 23:07:58 +0200 Subject: [PATCH 13/35] Add task. --- .../state_crawler/jobs/zksync-era-jobs.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json index 6301e903..1247343d 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json @@ -71,5 +71,20 @@ "address": "0xDFAaB828f5F515E104BaaBa4d8D554DA9096f0e4", "stateMutability": "view", "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "totalSupply", + "outputs": [ + { + "name": "", + "type": "uint256" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function", + "address": "0xDFAaB828f5F515E104BaaBa4d8D554DA9096f0e4" } ] \ No newline at end of file From 73df559c3e9e8400c499f070ff5af64de20e8b3d Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 14 Dec 2023 03:17:50 +0200 Subject: [PATCH 14/35] Add internal timeout. --- moonstreamapi/moonstreamapi/routes/queries.py | 3 ++- moonstreamapi/moonstreamapi/settings.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/moonstreamapi/moonstreamapi/routes/queries.py b/moonstreamapi/moonstreamapi/routes/queries.py index 5a1a0189..26e8b123 100644 --- a/moonstreamapi/moonstreamapi/routes/queries.py +++ b/moonstreamapi/moonstreamapi/routes/queries.py @@ -32,6 +32,7 @@ from ..settings import ( MOONSTREAM_APPLICATION_ID, MOONSTREAM_CRAWLERS_SERVER_PORT, MOONSTREAM_CRAWLERS_SERVER_URL, + MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS, MOONSTREAM_QUERIES_JOURNAL_ID, MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE, MOONSTREAM_S3_QUERIES_BUCKET, @@ -473,7 +474,7 @@ async def update_query_data_handler( if request_update.blockchain else None, }, - timeout=5, + timeout=MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS, ) except Exception as e: logger.error(f"Error interaction with crawlers: {str(e)}") diff --git a/moonstreamapi/moonstreamapi/settings.py b/moonstreamapi/moonstreamapi/settings.py index 95f7491a..1ae46003 100644 --- a/moonstreamapi/moonstreamapi/settings.py +++ b/moonstreamapi/moonstreamapi/settings.py @@ -259,3 +259,11 @@ supportsInterface_abi = [ "type": "function", } ] + +MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = os.environ.get( + "MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS", "10" +) + +MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = int( + MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS +) From 8bf0e723f0a14e2a927d4591f86c9899943ebd69 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 14 Dec 2023 03:35:55 +0200 Subject: [PATCH 15/35] Add suggested changes. --- moonstreamapi/moonstreamapi/settings.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/moonstreamapi/moonstreamapi/settings.py b/moonstreamapi/moonstreamapi/settings.py index 1ae46003..cf1db9a0 100644 --- a/moonstreamapi/moonstreamapi/settings.py +++ b/moonstreamapi/moonstreamapi/settings.py @@ -260,10 +260,16 @@ supportsInterface_abi = [ } ] -MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = os.environ.get( - "MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS", "10" -) - -MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = int( - MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS +MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS_RAW = os.environ.get( + "MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS" ) +MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = 10 +try: + if MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS_RAW is not None: + MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS = int( + MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS_RAW + ) +except: + raise Exception( + f"Could not parse MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS as int: {MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS_RAW}" + ) From b8b6d0c8c70d5c8c03fa109e1b3ed728ca2bf834 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Dec 2023 01:29:25 +0200 Subject: [PATCH 16/35] Clean deprecated workers. --- crawlers/deploy/deploy.bash | 42 +------------------ .../polygon-cu-reports-tokenonomics.service | 11 ----- .../polygon-cu-reports-tokenonomics.timer | 9 ---- crawlers/deploy/polygon-statistics.service | 11 ----- crawlers/deploy/polygon-statistics.timer | 9 ---- crawlers/deploy/wyrm-statistics.service | 11 ----- crawlers/deploy/wyrm-statistics.timer | 9 ---- crawlers/deploy/xdai-statistics.service | 11 ----- crawlers/deploy/xdai-statistics.timer | 9 ---- 9 files changed, 1 insertion(+), 121 deletions(-) delete mode 100644 crawlers/deploy/polygon-cu-reports-tokenonomics.service delete mode 100644 crawlers/deploy/polygon-cu-reports-tokenonomics.timer delete mode 100644 crawlers/deploy/polygon-statistics.service delete mode 100644 crawlers/deploy/polygon-statistics.timer delete mode 100644 crawlers/deploy/wyrm-statistics.service delete mode 100644 crawlers/deploy/wyrm-statistics.timer delete mode 100644 crawlers/deploy/xdai-statistics.service delete mode 100644 crawlers/deploy/xdai-statistics.timer diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 269a7cc8..b481dfd6 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -47,8 +47,6 @@ ETHEREUM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="ethereum-historical-crawl-events.ti POLYGON_SYNCHRONIZE_SERVICE="polygon-synchronize.service" POLYGON_MISSING_SERVICE_FILE="polygon-missing.service" POLYGON_MISSING_TIMER_FILE="polygon-missing.timer" -POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service" -POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer" POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service" POLYGON_STATE_SERVICE_FILE="polygon-state.service" POLYGON_STATE_TIMER_FILE="polygon-state.timer" @@ -56,8 +54,6 @@ POLYGON_STATE_CLEAN_SERVICE_FILE="polygon-state-clean.service" POLYGON_STATE_CLEAN_TIMER_FILE="polygon-state-clean.timer" POLYGON_METADATA_SERVICE_FILE="polygon-metadata.service" POLYGON_METADATA_TIMER_FILE="polygon-metadata.timer" -POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE="polygon-cu-reports-tokenonomics.service" -POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE="polygon-cu-reports-tokenonomics.timer" POLYGON_CU_NFT_DASHBOARD_SERVICE_FILE="polygon-cu-nft-dashboard.service" POLYGON_CU_NFT_DASHBOARD_TIMER_FILE="polygon-cu-nft-dashboard.timer" POLYGON_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="polygon-historical-crawl-transactions.service" @@ -86,8 +82,7 @@ MUMBAI_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="mumbai-historical-crawl-events.timer" XDAI_SYNCHRONIZE_SERVICE="xdai-synchronize.service" XDAI_MISSING_SERVICE_FILE="xdai-missing.service" XDAI_MISSING_TIMER_FILE="xdai-missing.timer" -XDAI_STATISTICS_SERVICE_FILE="xdai-statistics.service" -XDAI_STATISTICS_TIMER_FILE="xdai-statistics.timer" + XDAI_MOONWORM_CRAWLER_SERVICE_FILE="xdai-moonworm-crawler.service" XDai_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="xdai-historical-crawl-transactions.service" XDai_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="xdai-historical-crawl-transactions.timer" @@ -98,8 +93,6 @@ XDai_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="xdai-historical-crawl-events.timer" WYRM_SYNCHRONIZE_SERVICE="wyrm-synchronize.service" WYRM_MISSING_SERVICE_FILE="wyrm-missing.service" WYRM_MISSING_TIMER_FILE="wyrm-missing.timer" -WYRM_STATISTICS_SERVICE_FILE="wyrm-statistics.service" -WYRM_STATISTICS_TIMER_FILE="wyrm-statistics.timer" WYRM_MOONWORM_CRAWLER_SERVICE_FILE="wyrm-moonworm-crawler.service" WYRM_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="wyrm-historical-crawl-transactions.service" WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transactions.timer" @@ -256,14 +249,6 @@ cp "${SCRIPT_DIR}/${POLYGON_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/u XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_MISSING_TIMER_FILE}" -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Polygon statistics dashbord service and timer with: ${POLYGON_STATISTICS_SERVICE_FILE}, ${POLYGON_STATISTICS_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${POLYGON_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" -cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_STATISTICS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_STATISTICS_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}" echo echo @@ -300,14 +285,6 @@ cp "${SCRIPT_DIR}/${POLYGON_METADATA_TIMER_FILE}" "/home/ubuntu/.config/systemd/ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_METADATA_TIMER_FILE}" -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Polygon CU reports tokenonomics service and timer with: ${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}, ${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" -cp "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_CU_REPORTS_TOKENONOMICS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_CU_REPORTS_TOKENONOMICS_TIMER_FILE}" echo echo @@ -423,14 +400,6 @@ cp "${SCRIPT_DIR}/${XDAI_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XDAI_MISSING_TIMER_FILE}" -echo -echo -echo -e "${PREFIX_INFO} Replacing existing XDai statistics dashbord service and timer with: ${XDAI_STATISTICS_SERVICE_FILE}, ${XDAI_STATISTICS_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${XDAI_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${XDAI_STATISTICS_TIMER_FILE}" -cp "${SCRIPT_DIR}/${XDAI_STATISTICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XDAI_STATISTICS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${XDAI_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XDAI_STATISTICS_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XDAI_STATISTICS_TIMER_FILE}" echo echo @@ -475,15 +444,6 @@ cp "${SCRIPT_DIR}/${WYRM_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_MISSING_TIMER_FILE}" -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Wyrm statistics dashbord service and timer with: ${WYRM_STATISTICS_SERVICE_FILE}, ${WYRM_STATISTICS_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${WYRM_STATISTICS_SERVICE_FILE}" "${SCRIPT_DIR}/${WYRM_STATISTICS_TIMER_FILE}" -cp "${SCRIPT_DIR}/${WYRM_STATISTICS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_STATISTICS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${WYRM_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_STATISTICS_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_STATISTICS_TIMER_FILE}" - echo echo echo -e "${PREFIX_INFO} Replacing existing Wyrm moonworm crawler service definition with ${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}" diff --git a/crawlers/deploy/polygon-cu-reports-tokenonomics.service b/crawlers/deploy/polygon-cu-reports-tokenonomics.service deleted file mode 100644 index dededd71..00000000 --- a/crawlers/deploy/polygon-cu-reports-tokenonomics.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Runs custom crawler for CU tokenonomics -After=network.target - -[Service] -Type=oneshot -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.reports_crawler.cli cu-reports --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" queries run-tokenonomics -CPUWeight=60 -SyslogIdentifier=polygon-cu-reports-tokenonomics \ No newline at end of file diff --git a/crawlers/deploy/polygon-cu-reports-tokenonomics.timer b/crawlers/deploy/polygon-cu-reports-tokenonomics.timer deleted file mode 100644 index 3a56d85d..00000000 --- a/crawlers/deploy/polygon-cu-reports-tokenonomics.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Runs custom crawler for CU tokenonomics - -[Timer] -OnBootSec=60s -OnUnitActiveSec=60m - -[Install] -WantedBy=timers.target diff --git a/crawlers/deploy/polygon-statistics.service b/crawlers/deploy/polygon-statistics.service deleted file mode 100644 index ea51aa85..00000000 --- a/crawlers/deploy/polygon-statistics.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Update Polygon statistics dashboards -After=network.target - -[Service] -Type=oneshot -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard --access-id "${NB_CONTROLLER_ACCESS_ID}" generate --blockchain polygon -CPUWeight=60 -SyslogIdentifier=polygon-statistics \ No newline at end of file diff --git a/crawlers/deploy/polygon-statistics.timer b/crawlers/deploy/polygon-statistics.timer deleted file mode 100644 index 0cab38da..00000000 --- a/crawlers/deploy/polygon-statistics.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Update Polygon statistics dashboards each 6 hours - -[Timer] -OnBootSec=20s -OnUnitActiveSec=6h - -[Install] -WantedBy=timers.target diff --git a/crawlers/deploy/wyrm-statistics.service b/crawlers/deploy/wyrm-statistics.service deleted file mode 100644 index e8a7add2..00000000 --- a/crawlers/deploy/wyrm-statistics.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Update Wyrm statistics dashboards -After=network.target - -[Service] -Type=oneshot -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard --access-id "${NB_CONTROLLER_ACCESS_ID}" generate --blockchain wyrm -CPUWeight=60 -SyslogIdentifier=wyrm-statistics \ No newline at end of file diff --git a/crawlers/deploy/wyrm-statistics.timer b/crawlers/deploy/wyrm-statistics.timer deleted file mode 100644 index 885c5191..00000000 --- a/crawlers/deploy/wyrm-statistics.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Update Wyrm statistics dashboards each 6 hours - -[Timer] -OnBootSec=25s -OnUnitActiveSec=6h - -[Install] -WantedBy=timers.target \ No newline at end of file diff --git a/crawlers/deploy/xdai-statistics.service b/crawlers/deploy/xdai-statistics.service deleted file mode 100644 index 4a1b12cb..00000000 --- a/crawlers/deploy/xdai-statistics.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Update XDai statistics dashboards -After=network.target - -[Service] -Type=oneshot -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.stats_worker.dashboard --access-id "${NB_CONTROLLER_ACCESS_ID}" generate --blockchain xdai -CPUWeight=60 -SyslogIdentifier=xdai-statistics \ No newline at end of file diff --git a/crawlers/deploy/xdai-statistics.timer b/crawlers/deploy/xdai-statistics.timer deleted file mode 100644 index 8ef61b51..00000000 --- a/crawlers/deploy/xdai-statistics.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Update XDai statistics dashboards each 6 hours - -[Timer] -OnBootSec=25s -OnUnitActiveSec=6h - -[Install] -WantedBy=timers.target \ No newline at end of file From ad9e332d15ca7d16fc1dae1f79d7e7a14b72148a Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Dec 2023 04:06:46 +0200 Subject: [PATCH 17/35] Fix leaderboard info endpoint. --- engineapi/engineapi/actions.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index ef7f79b6..c85e8059 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1029,7 +1029,7 @@ def get_leaderboard_info( version_number=version_number, ) - leaderboard = ( + query = ( db_session.query( Leaderboard.id, Leaderboard.title, @@ -1052,12 +1052,19 @@ def get_leaderboard_info( isouter=True, ) .filter( - LeaderboardVersion.published == True, - LeaderboardVersion.version_number == latest_version, + or_( + LeaderboardVersion.published == None, + and_( + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, + ), + ) ) .filter(Leaderboard.id == leaderboard_id) .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) - ).one() + ) + + leaderboard = query.one() return leaderboard From 919c7c6a79c404da25ca54ff00cd836be32dc7aa Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Dec 2023 04:08:34 +0200 Subject: [PATCH 18/35] revert changes. --- engineapi/engineapi/actions.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index c85e8059..ef7f79b6 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1029,7 +1029,7 @@ def get_leaderboard_info( version_number=version_number, ) - query = ( + leaderboard = ( db_session.query( Leaderboard.id, Leaderboard.title, @@ -1052,19 +1052,12 @@ def get_leaderboard_info( isouter=True, ) .filter( - or_( - LeaderboardVersion.published == None, - and_( - LeaderboardVersion.published == True, - LeaderboardVersion.version_number == latest_version, - ), - ) + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, ) .filter(Leaderboard.id == leaderboard_id) .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) - ) - - leaderboard = query.one() + ).one() return leaderboard From 4d90055813abe38c908baa3640670dd26f49326b Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Dec 2023 04:51:20 +0200 Subject: [PATCH 19/35] Fix leaderboard info endpoint. --- engineapi/engineapi/actions.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index ef7f79b6..5d8071af 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1029,7 +1029,7 @@ def get_leaderboard_info( version_number=version_number, ) - leaderboard = ( + query = ( db_session.query( Leaderboard.id, Leaderboard.title, @@ -1038,26 +1038,33 @@ def get_leaderboard_info( func.max(LeaderboardScores.updated_at).label("last_update"), ) .join( - LeaderboardScores, - LeaderboardScores.leaderboard_id == Leaderboard.id, + LeaderboardVersion, + and_( + LeaderboardVersion.leaderboard_id == Leaderboard.id, + LeaderboardVersion.published == True, + ), isouter=True, ) .join( - LeaderboardVersion, + LeaderboardScores, and_( - LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, - LeaderboardVersion.version_number - == LeaderboardScores.leaderboard_version_number, + LeaderboardScores.leaderboard_id == Leaderboard.id, + LeaderboardScores.leaderboard_version_number + == LeaderboardVersion.version_number, ), isouter=True, ) .filter( - LeaderboardVersion.published == True, - LeaderboardVersion.version_number == latest_version, + or_( + LeaderboardVersion.published == None, + LeaderboardVersion.version_number == latest_version, + ) ) .filter(Leaderboard.id == leaderboard_id) .group_by(Leaderboard.id, Leaderboard.title, Leaderboard.description) - ).one() + ) + + leaderboard = query.one() return leaderboard From 7a24e2fb85c3417d11851fb8268eab0b23306d31 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Dec 2023 05:01:57 +0200 Subject: [PATCH 20/35] remove new line. --- crawlers/deploy/deploy.bash | 1 - 1 file changed, 1 deletion(-) diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index b481dfd6..9f883381 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -82,7 +82,6 @@ MUMBAI_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="mumbai-historical-crawl-events.timer" XDAI_SYNCHRONIZE_SERVICE="xdai-synchronize.service" XDAI_MISSING_SERVICE_FILE="xdai-missing.service" XDAI_MISSING_TIMER_FILE="xdai-missing.timer" - XDAI_MOONWORM_CRAWLER_SERVICE_FILE="xdai-moonworm-crawler.service" XDai_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="xdai-historical-crawl-transactions.service" XDai_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="xdai-historical-crawl-transactions.timer" From cfeae159d4e34616e1e6996a94f8875a0a9b247e Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 10 Jan 2024 17:25:30 +0200 Subject: [PATCH 21/35] Add get score endpoint. --- engineapi/engineapi/actions.py | 37 ++++++++++++++++ engineapi/engineapi/routes/leaderboard.py | 52 +++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index ef7f79b6..28077dd7 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1222,6 +1222,43 @@ def get_position( return query.all() +def get_leaderboard_score( + db_session: Session, + leaderboard_id, + address, + version_number: Optional[int] = None, +) -> Optional[LeaderboardScores]: + """ + Return address score + """ + + latest_version = leaderboard_version_filter( + db_session=db_session, + leaderboard_id=leaderboard_id, + version_number=version_number, + ) + + query = ( + db_session.query(LeaderboardScores) + .join( + LeaderboardVersion, + and_( + LeaderboardVersion.leaderboard_id == LeaderboardScores.leaderboard_id, + LeaderboardVersion.version_number + == LeaderboardScores.leaderboard_version_number, + ), + ) + .filter( + LeaderboardVersion.published == True, + LeaderboardVersion.version_number == latest_version, + ) + .filter(LeaderboardScores.leaderboard_id == leaderboard_id) + .filter(LeaderboardScores.address == address) + ) + + return query.one_or_none() + + def get_leaderboard_positions( db_session: Session, leaderboard_id, diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 20c6e0dd..e18f9957 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -60,6 +60,7 @@ leaderboard_whitelist = { "/leaderboard/": "GET", "/leaderboard/rank": "GET", "/leaderboard/ranks": "GET", + "/leaderboard/scores": "GET", "/scores/changes": "GET", "/leaderboard/docs": "GET", "/leaderboard/openapi.json": "GET", @@ -619,6 +620,57 @@ async def ranks( return results +@app.get( + "/scores", + response_model=data.LeaderboardScore, + tags=["Public Endpoints"], +) +async def leaderboard_score( + address: str = Query(..., description="Address to get position for."), + leaderboard_id: UUID = Query(..., description="Leaderboard ID"), + version: Optional[int] = Query(None, description="Version of the leaderboard."), + normalize_addresses: bool = Query( + True, description="Normalize addresses to checksum." + ), + db_session: Session = Depends(db.yield_db_session), +) -> data.LeaderboardScore: + """ + Returns the leaderboard posotion for the given address. + """ + + ### Check if leaderboard exists + try: + actions.get_leaderboard_by_id(db_session, leaderboard_id) + except NoResultFound as e: + raise EngineHTTPException( + status_code=404, + detail="Leaderboard not found.", + ) + except Exception as e: + logger.error(f"Error while getting leaderboard: {e}") + raise EngineHTTPException(status_code=500, detail="Internal server error") + + if normalize_addresses: + address = Web3.toChecksumAddress(address) + + score = actions.get_leaderboard_score( + db_session, + leaderboard_id, + address, + version, + ) + + if score is None: + raise EngineHTTPException(status_code=204, detail="Score not found.") + + return data.LeaderboardScore( + leaderboard_id=score.leaderboard_id, + address=score.address, + score=score.score, + points_data=score.points_data, + ) + + @app.put( "/{leaderboard_id}/scores", response_model=List[data.LeaderboardScore], From 00a5b21505d5fb9b8ba583fd2a6bab92c495c381 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 11 Jan 2024 09:41:01 +0000 Subject: [PATCH 22/35] Fixed monitoring tag for deployment --- crawlers/deploy/deploy-monitoring.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crawlers/deploy/deploy-monitoring.bash b/crawlers/deploy/deploy-monitoring.bash index 12600b48..48f4c773 100755 --- a/crawlers/deploy/deploy-monitoring.bash +++ b/crawlers/deploy/deploy-monitoring.bash @@ -39,7 +39,7 @@ chown ubuntu:ubuntu "/home/ubuntu/monitoring" echo echo echo -e "${PREFIX_INFO} Retrieving monitoring deployment parameters" -AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+crawlers:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}" +AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+service:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}" chmod 0640 "${PARAMETERS_ENV_MONITORING_PATH}" echo @@ -49,7 +49,7 @@ echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_MONITORIN echo echo -echo -e "${PREFIX_INFO} Add AWS default region to monitring parameters" +echo -e "${PREFIX_INFO} Add AWS default region to monitoring parameters" echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> "${PARAMETERS_ENV_MONITORING_PATH}" echo From a3745888aef079295738eb55986a520db86a4ef2 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 23 Jan 2024 06:32:18 +0000 Subject: [PATCH 23/35] Latest block check for starknet goerli --- nodebalancer/cmd/nodebalancer/balancer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebalancer/cmd/nodebalancer/balancer.go b/nodebalancer/cmd/nodebalancer/balancer.go index d8dea3c4..701d6dd2 100644 --- a/nodebalancer/cmd/nodebalancer/balancer.go +++ b/nodebalancer/cmd/nodebalancer/balancer.go @@ -198,7 +198,7 @@ func (bpool *BlockchainPool) HealthCheck() { for _, b := range bpool.Blockchains { var timeout time.Duration getLatestBlockReq := `{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest", false],"id":1}` - if b.Blockchain == "starknet" { + if b.Blockchain == "starknet" || b.Blockchain == "starknet-goerli" { getLatestBlockReq = `{"jsonrpc":"2.0","method":"starknet_getBlockWithTxHashes","params":["latest"],"id":"0"}` timeout = NB_HEALTH_CHECK_CALL_TIMEOUT * 2 } From 03c3a2bc345a4abcc41b9cdf28849a268a27e817 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 23 Jan 2024 06:45:11 +0000 Subject: [PATCH 24/35] Fix starknet goerli latest check for nb --- nodebalancer/cmd/nodebalancer/balancer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebalancer/cmd/nodebalancer/balancer.go b/nodebalancer/cmd/nodebalancer/balancer.go index 701d6dd2..1496b049 100644 --- a/nodebalancer/cmd/nodebalancer/balancer.go +++ b/nodebalancer/cmd/nodebalancer/balancer.go @@ -241,7 +241,7 @@ func (bpool *BlockchainPool) HealthCheck() { } var blockNumber uint64 - if b.Blockchain == "starknet" { + if b.Blockchain == "starknet" || b.Blockchain == "starknet-goerli" { blockNumber = statusResponse.Result.BlockNumber } else { blockNumberHex := strings.Replace(statusResponse.Result.Number, "0x", "", -1) From b7aa897c50b5907a91fc2d9341381788d1ac6a95 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 23 Jan 2024 07:01:23 +0000 Subject: [PATCH 25/35] Env for healthcheck interval for nb --- nodebalancer/cmd/nodebalancer/configs.go | 2 +- nodebalancer/cmd/nodebalancer/server.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/nodebalancer/cmd/nodebalancer/configs.go b/nodebalancer/cmd/nodebalancer/configs.go index 49ffdd00..e606dc83 100644 --- a/nodebalancer/cmd/nodebalancer/configs.go +++ b/nodebalancer/cmd/nodebalancer/configs.go @@ -38,7 +38,7 @@ var ( NB_CONNECTION_RETRIES = 2 NB_CONNECTION_RETRIES_INTERVAL = time.Millisecond * 10 - NB_HEALTH_CHECK_INTERVAL = time.Millisecond * 5000 + NB_HEALTH_CHECK_INTERVAL = os.Getenv("NB_HEALTH_CHECK_INTERVAL") NB_HEALTH_CHECK_CALL_TIMEOUT = time.Second * 2 NB_CACHE_CLEANING_INTERVAL = time.Second * 10 diff --git a/nodebalancer/cmd/nodebalancer/server.go b/nodebalancer/cmd/nodebalancer/server.go index 648fa3da..e1cbe155 100644 --- a/nodebalancer/cmd/nodebalancer/server.go +++ b/nodebalancer/cmd/nodebalancer/server.go @@ -12,6 +12,7 @@ import ( "net/http/httputil" "net/url" "os" + "strconv" "strings" "time" @@ -28,7 +29,11 @@ var ( // initHealthCheck runs a routine for check status of the nodes every 5 seconds func initHealthCheck(debug bool) { - t := time.NewTicker(NB_HEALTH_CHECK_INTERVAL) + healthCheckInterval, convErr := strconv.Atoi(NB_HEALTH_CHECK_INTERVAL) + if convErr != nil { + healthCheckInterval = 5 + } + t := time.NewTicker(time.Second * time.Duration(healthCheckInterval)) for { select { case <-t.C: From 52aafc328bf5b47603c8ee625a2cdf0cd91e4e11 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 25 Jan 2024 05:44:38 +0200 Subject: [PATCH 26/35] Remove user interactions with resources(only list). --- moonstreamapi/moonstreamapi/actions.py | 55 +++++++++++++++++++ moonstreamapi/moonstreamapi/routes/queries.py | 35 +++++------- 2 files changed, 70 insertions(+), 20 deletions(-) diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index 841af7b5..0c3b68c5 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -15,6 +15,9 @@ from bugout.data import ( BugoutResources, BugoutSearchResult, BugoutSearchResults, + BugoutResourceHolder, + HolderType, + ResourcePermissions, ) from bugout.exceptions import BugoutResponseException from bugout.journal import SearchOrder @@ -952,3 +955,55 @@ def check_if_smart_contract( is_contract = True return blockchain_type, address, is_contract + + +def create_resource_for_user( + user_id: uuid.UUID, + resource_data: Dict[str, Any], +) -> BugoutResource: + """ + Create resource for user + """ + try: + resource = bc.create_resource( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + application_id=MOONSTREAM_APPLICATION_ID, + resource_data=resource_data, + ) + except BugoutResponseException as e: + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + logger.error(f"Error creating resource: {str(e)}") + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + try: + bc.add_resource_holder_permissions( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + resource_id=resource.id, + holder_permissions=BugoutResourceHolder( + holder_type=HolderType.user, + holder_id=user_id, + permission_list=[ + ResourcePermissions.ADMIN, + ResourcePermissions.READ, + ResourcePermissions.UPDATE, + ResourcePermissions.DELETE, + ], + ), + ) + except BugoutResponseException as e: + logger.error( + f"Error adding resource holder permissions to resource resource {str(resource.id)} {str(e)}" + ) + raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) + except Exception as e: + bc.delete_resource( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + resource_id=resource.id, + ) + logger.error( + f"Error adding resource holder permissions to resource {str(resource.id)} {str(e)}" + ) + raise MoonstreamHTTPException(status_code=500, internal_error=e) + + return resource diff --git a/moonstreamapi/moonstreamapi/routes/queries.py b/moonstreamapi/moonstreamapi/routes/queries.py index 26e8b123..42d1277c 100644 --- a/moonstreamapi/moonstreamapi/routes/queries.py +++ b/moonstreamapi/moonstreamapi/routes/queries.py @@ -25,6 +25,7 @@ from ..actions import ( get_query_by_name, name_normalization, query_parameter_hash, + create_resource_for_user, ) from ..middleware import MoonstreamHTTPException from ..settings import ( @@ -130,24 +131,16 @@ async def create_query_handler( except Exception as e: raise MoonstreamHTTPException(status_code=500, internal_error=e) - try: - # create resource query_name_resolver - bc.create_resource( - token=token, - application_id=MOONSTREAM_APPLICATION_ID, - resource_data={ - "type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, - "user_id": str(user.id), - "user": str(user.username), - "name": query_name, - "entry_id": str(entry.id), - }, - ) - except BugoutResponseException as e: - logger.error(f"Error creating name resolving resource: {str(e)}") - raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) - except Exception as e: - raise MoonstreamHTTPException(status_code=500, internal_error=e) + create_resource_for_user( + user_id=user.id, + resource_data={ + "type": data.BUGOUT_RESOURCE_QUERY_RESOLVER, + "user_id": str(user.id), + "user": str(user.username), + "name": query_name, + "entry_id": str(entry.id), + }, + ) try: bc.update_tags( @@ -355,7 +348,7 @@ async def update_query_handler( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, journal_id=MOONSTREAM_QUERIES_JOURNAL_ID, entry_id=query_id, - title=query_name, + title=f"Query:{query_name}", content=request_update.query, tags=["preapprove"], ) @@ -620,7 +613,9 @@ async def remove_query_handler( raise MoonstreamHTTPException(status_code=404, detail="Query does not exists") try: - bc.delete_resource(token=token, resource_id=query_ids[query_name][0]) + bc.delete_resource( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, resource_id=query_ids[query_name][0] + ) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: From e7be016c980e5c27f83d7893335cc8e0b6499be8 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 25 Jan 2024 06:25:18 +0200 Subject: [PATCH 27/35] Fix engine and subscriptions. --- engineapi/engineapi/actions.py | 66 ++++++++++++++----- moonstreamapi/moonstreamapi/actions.py | 6 +- moonstreamapi/moonstreamapi/routes/queries.py | 1 - 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 2e8f2f38..38537ab3 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -5,7 +5,13 @@ from typing import List, Any, Optional, Dict, Union, Tuple, cast import uuid import logging -from bugout.data import BugoutResource, BugoutSearchResult +from bugout.data import ( + BugoutResource, + BugoutSearchResult, + ResourcePermissions, + HolderType, + BugoutResourceHolder, +) from eth_typing import Address from hexbytes import HexBytes import requests # type: ignore @@ -96,6 +102,10 @@ class LeaderboardVersionNotFound(Exception): pass +class LeaderboardAssignResourceError(Exception): + pass + + BATCH_SIGNATURE_PAGE_SIZE = 500 logger = logging.getLogger(__name__) @@ -1486,26 +1496,25 @@ def create_leaderboard( Create a leaderboard """ - if not token: - token = uuid.UUID(MOONSTREAM_ADMIN_ACCESS_TOKEN) try: leaderboard = Leaderboard(title=title, description=description) db_session.add(leaderboard) db_session.commit() + user = None + if token is not None: + user = bc.get_user(token=token) + resource = create_leaderboard_resource( leaderboard_id=str(leaderboard.id), - token=token, + user_id=str(user.id) if user is not None else None, ) - leaderboard.resource_id = resource.id - db_session.commit() except Exception as e: db_session.rollback() logger.error(f"Error creating leaderboard: {e}") raise LeaderboardCreateError(f"Error creating leaderboard: {e}") - return leaderboard @@ -1659,38 +1668,62 @@ def add_scores( # leaderboard access actions -def create_leaderboard_resource( - leaderboard_id: str, token: Union[Optional[uuid.UUID], str] = None -) -> BugoutResource: +def create_leaderboard_resource(leaderboard_id: str, user_id: Optional[str] = None): resource_data: Dict[str, Any] = { "type": LEADERBOARD_RESOURCE_TYPE, "leaderboard_id": leaderboard_id, } - if token is None: - token = MOONSTREAM_ADMIN_ACCESS_TOKEN try: resource = bc.create_resource( - token=token, + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, application_id=MOONSTREAM_APPLICATION_ID, resource_data=resource_data, timeout=10, ) except Exception as e: raise LeaderboardCreateError(f"Error creating leaderboard resource: {e}") + + if user_id is not None: + try: + bc.add_resource_holder_permissions( + token=MOONSTREAM_ADMIN_ACCESS_TOKEN, + resource_id=resource.id, + holder_permissions=BugoutResourceHolder( + holder_type=HolderType.user, + holder_id=user_id, + permission_list=[ + ResourcePermissions.ADMIN, + ResourcePermissions.READ, + ResourcePermissions.UPDATE, + ResourcePermissions.DELETE, + ], + ), + ) + except Exception as e: + raise LeaderboardCreateError( + f"Error adding resource holder permissions: {e}" + ) + return resource def assign_resource( db_session: Session, leaderboard_id: uuid.UUID, - user_token: Union[uuid.UUID, str], + user_token: Optional[Union[uuid.UUID, str]] = None, resource_id: Optional[uuid.UUID] = None, ): """ Assign a resource handler to a leaderboard """ + ### get user_name from token + + user = None + if user_token is not None: + user = bc.get_user(token=user_token) + leaderboard = ( db_session.query(Leaderboard).filter(Leaderboard.id == leaderboard_id).one() # type: ignore ) @@ -1698,11 +1731,8 @@ def assign_resource( if resource_id is not None: leaderboard.resource_id = resource_id else: - # Create resource via admin token - resource = create_leaderboard_resource( - leaderboard_id=str(leaderboard_id), - token=user_token, + leaderboard_id=str(leaderboard_id), user_id=user.id ) leaderboard.resource_id = resource.id diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index 0c3b68c5..ad9f7c22 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -714,11 +714,7 @@ def generate_journal_for_user( } try: - bc.create_resource( - token=token, - application_id=MOONSTREAM_APPLICATION_ID, - resource_data=resource_data, - ) + create_resource_for_user(user_id=user_id, resource_data=resource_data) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) except Exception as e: diff --git a/moonstreamapi/moonstreamapi/routes/queries.py b/moonstreamapi/moonstreamapi/routes/queries.py index 42d1277c..90fa5636 100644 --- a/moonstreamapi/moonstreamapi/routes/queries.py +++ b/moonstreamapi/moonstreamapi/routes/queries.py @@ -30,7 +30,6 @@ from ..actions import ( from ..middleware import MoonstreamHTTPException from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, - MOONSTREAM_APPLICATION_ID, MOONSTREAM_CRAWLERS_SERVER_PORT, MOONSTREAM_CRAWLERS_SERVER_URL, MOONSTREAM_INTERNAL_REQUEST_TIMEOUT_SECONDS, From 613f8abc0cdb64f0ea59b8fd01a9d271cd792f79 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 27 Jan 2024 09:28:45 +0200 Subject: [PATCH 28/35] Leaderboard versions simplify. --- .../71e888082a6d_leaderboard_metadata.py | 2 +- engineapi/engineapi/actions.py | 24 ++- engineapi/engineapi/data.py | 30 ++- engineapi/engineapi/models.py | 2 +- engineapi/engineapi/routes/leaderboard.py | 182 +++++++----------- 5 files changed, 102 insertions(+), 138 deletions(-) diff --git a/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py b/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py index 43ef9d0c..37ad0604 100644 --- a/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py +++ b/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py @@ -38,7 +38,7 @@ def upgrade(): sa.Column( "columns_names", postgresql.JSONB(astext_type=sa.Text()), - nullable=False, + nullable=True, server_default="{}", ), ) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index e845cc39..a1bef295 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -22,7 +22,7 @@ from .data import ( LeaderboardConfigUpdate, LeaderboardConfig, LeaderboardPosition, - LeaderboardUnformattedPosition, + ColumnsNames, ) from .contracts import Dropper_interface, ERC20_interface, Terminus_interface from .models import ( @@ -1296,7 +1296,7 @@ def get_leaderboard_score( def get_leaderboard_positions( db_session: Session, - leaderboard_id, + leaderboard_id: uuid.UUID, limit: int, offset: int, version_number: Optional[int] = None, @@ -1511,14 +1511,14 @@ def create_leaderboard( token: Optional[Union[uuid.UUID, str]] = None, wallet_connect: bool = False, blockchain_ids: List[int] = [], - columns_names: Optional[Dict[str, str]] = {}, + columns_names: ColumnsNames = None, ) -> Leaderboard: """ Create a leaderboard """ - if len(columns_names) > 0: - columns_names = get_default_columns_names(columns_names) + if columns_names is not None: + columns_names = columns_names.dict() if not token: token = uuid.UUID(MOONSTREAM_ADMIN_ACCESS_TOKEN) @@ -1593,7 +1593,8 @@ def update_leaderboard( description: Optional[str], wallet_connect: Optional[bool], blockchain_ids: Optional[List[int]], - columns_names: Optional[Dict[str, str]], + columns_names: Optional[ColumnsNames], + delete_names, ) -> Leaderboard: """ Update a leaderboard @@ -1615,8 +1616,15 @@ def update_leaderboard( leaderboard.blockchain_ids = blockchain_ids if columns_names is not None: - columns_names = get_default_columns_names(columns_names) - leaderboard.columns_names = columns_names + if leaderboard.columns_names is not None: + current_columns_names = ColumnsNames(**leaderboard.columns_names) + + for key, value in columns_names.dict(exclude_none=True).items(): + setattr(current_columns_names, key, value) + else: + current_columns_names = columns_names + + leaderboard.columns_names = current_columns_names.dict() db_session.commit() diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 30bcb639..99fadbec 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -342,16 +342,6 @@ class Score(BaseModel): points_data: Dict[str, Any] -class LeaderboardUnformattedPosition(BaseModel): - column_1: str = Field(serialization_alias="address") - column_2: int = Field(serialization_alias="rank") - column_3: int = Field(serialization_alias="score") - column_4: Dict[str, Any] = Field(serialization_alias="points_data") - - class Config: - orm_mode = True - - class LeaderboardPosition(BaseModel): address: str rank: int @@ -375,6 +365,14 @@ class LeaderboardScore(BaseModel): points_data: Dict[str, Any] +class ColumnsNames(BaseModel): + rank: Optional[str] = None + address: Optional[str] = None + score: Optional[str] = None + points_data: Optional[str] = None + points_data_fields: Dict[str, str] = Field(default_factory=dict) + + class Leaderboard(BaseModel): id: UUID title: str @@ -382,7 +380,7 @@ class Leaderboard(BaseModel): resource_id: Optional[UUID] = None wallet_connect: bool = False blockchain_ids: List[int] = Field(default_factory=list) - columns_names: Dict[str, str] = Field(default_factory=dict) + columns_names: ColumnsNames = Field(default_factory=ColumnsNames) created_at: datetime updated_at: datetime @@ -400,7 +398,7 @@ class LeaderboardCreateRequest(BaseModel): description: Optional[str] = None wallet_connect: bool = False blockchain_ids: List[int] = Field(default_factory=list) - columns_names: Dict[str, str] = Field(default_factory=dict) + columns_names: Optional[ColumnsNames] = None class LeaderboardCreatedResponse(BaseModel): @@ -410,7 +408,7 @@ class LeaderboardCreatedResponse(BaseModel): resource_id: Optional[UUID] = None wallet_connect: bool = False blockchain_ids: List[int] = Field(default_factory=list) - columns_names: Dict[str, str] = Field(default_factory=dict) + columns_names: Optional[ColumnsNames] = None created_at: datetime updated_at: datetime @@ -425,7 +423,7 @@ class LeaderboardUpdatedResponse(BaseModel): resource_id: Optional[UUID] = None wallet_connect: bool = False blockchain_ids: List[int] = Field(default_factory=list) - columns_names: Dict[str, str] = Field(default_factory=dict) + columns_names: Optional[ColumnsNames] = None created_at: datetime updated_at: datetime @@ -438,7 +436,7 @@ class LeaderboardUpdateRequest(BaseModel): description: Optional[str] = None wallet_connect: bool = False blockchain_ids: List[int] = Field(default_factory=list) - columns_names: Dict[str, str] = Field(default_factory=dict) + columns_names: Optional[ColumnsNames] = None class LeaderboardDeletedResponse(BaseModel): @@ -448,7 +446,7 @@ class LeaderboardDeletedResponse(BaseModel): resource_id: Optional[UUID] = None wallet_connect: bool = False blockchain_ids: List[int] = Field(default_factory=list) - columns_names: Dict[str, str] = Field(default_factory=dict) + columns_names: Optional[ColumnsNames] = None created_at: datetime updated_at: datetime diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index b03b7e33..febb5679 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -350,7 +350,7 @@ class Leaderboard(Base): # type: ignore blockchain_ids = Column(ARRAY(Integer), nullable=False, default=[]) wallet_connect = Column(Boolean, default=False, nullable=False) - columns_names = Column(JSONB, nullable=False, default={}) + columns_names = Column(JSONB, nullable=True, default={}) created_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False ) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 2107a994..9861044c 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -90,16 +90,12 @@ app.add_middleware( @app.get( "", - response_model=Union[ - List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition] - ], + response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"], ) @app.get( "/", - response_model=Union[ - List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition] - ], + response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"], ) async def leaderboard( @@ -126,29 +122,18 @@ async def leaderboard( raise EngineHTTPException(status_code=500, detail="Internal server error") leaderboard_positions = actions.get_leaderboard_positions( - db_session, leaderboard_id, limit, offset, version + db_session, leaderboard.id, limit, offset, version ) - if len(leaderboard.columns_names) > 0: - result = [ - data.LeaderboardUnformattedPosition( - column_1=position[1], - column_2=position[2], - column_3=position[4], - column_4=position[3], - ) - for position in leaderboard_positions - ] - else: - result = [ - data.LeaderboardPosition( - address=position.address, - score=position.score, - rank=position.rank, - points_data=position.points_data, - ) - for position in leaderboard_positions - ] + result = [ + data.LeaderboardPosition( + address=position.address, + score=position.score, + rank=position.rank, + points_data=position.points_data, + ) + for position in leaderboard_positions + ] return result @@ -198,15 +183,15 @@ async def create_leaderboard( # Add resource to the leaderboard return data.LeaderboardCreatedResponse( - id=created_leaderboard.id, # type: ignore - title=created_leaderboard.title, # type: ignore - description=created_leaderboard.description, # type: ignore - resource_id=created_leaderboard.resource_id, # type: ignore - wallet_connect=created_leaderboard.wallet_connect, # type: ignore - blockchain_ids=created_leaderboard.blockchain_ids, # type: ignore - columns_names=created_leaderboard.columns_names, # type: ignore - created_at=created_leaderboard.created_at, # type: ignore - updated_at=created_leaderboard.updated_at, # type: ignore + id=created_leaderboard.id, + title=created_leaderboard.title, + description=created_leaderboard.description, + resource_id=created_leaderboard.resource_id, + wallet_connect=created_leaderboard.wallet_connect, + blockchain_ids=created_leaderboard.blockchain_ids, + columns_names=created_leaderboard.columns_names, + created_at=created_leaderboard.created_at, + updated_at=created_leaderboard.updated_at, ) @@ -266,15 +251,15 @@ async def update_leaderboard( raise EngineHTTPException(status_code=500, detail="Internal server error") return data.LeaderboardUpdatedResponse( - id=updated_leaderboard.id, # type: ignore - title=updated_leaderboard.title, # type: ignore - description=updated_leaderboard.description, # type: ignore - resource_id=updated_leaderboard.resource_id, # type: ignore - wallet_connect=updated_leaderboard.wallet_connect, # type: ignore - blockchain_ids=updated_leaderboard.blockchain_ids, # type: ignore - columns_names=updated_leaderboard.columns_names, # type: ignore - created_at=updated_leaderboard.created_at, # type: ignore - updated_at=updated_leaderboard.updated_at, # type: ignore + id=updated_leaderboard.id, + title=updated_leaderboard.title, + description=updated_leaderboard.description, + resource_id=updated_leaderboard.resource_id, + wallet_connect=updated_leaderboard.wallet_connect, + blockchain_ids=updated_leaderboard.blockchain_ids, + columns_names=updated_leaderboard.columns_names, + created_at=updated_leaderboard.created_at, + updated_at=updated_leaderboard.updated_at, ) @@ -329,15 +314,15 @@ async def delete_leaderboard( raise EngineHTTPException(status_code=500, detail="Internal server error") return data.LeaderboardDeletedResponse( - id=deleted_leaderboard.id, # type: ignore - title=deleted_leaderboard.title, # type: ignore - description=deleted_leaderboard.description, # type: ignore - resource_id=deleted_leaderboard.resource_id, # type: ignore - wallet_connect=deleted_leaderboard.wallet_connect, # type: ignore - blockchain_ids=deleted_leaderboard.blockchain_ids, # type: ignore - columns_names=deleted_leaderboard.columns_names, # type: ignore - created_at=deleted_leaderboard.created_at, # type: ignore - updated_at=deleted_leaderboard.updated_at, # type: ignore + id=deleted_leaderboard.id, + title=deleted_leaderboard.title, + description=deleted_leaderboard.description, + resource_id=deleted_leaderboard.resource_id, + wallet_connect=deleted_leaderboard.wallet_connect, + blockchain_ids=deleted_leaderboard.blockchain_ids, + columns_names=deleted_leaderboard.columns_names, + created_at=deleted_leaderboard.created_at, + updated_at=deleted_leaderboard.updated_at, ) @@ -370,15 +355,15 @@ async def get_leaderboards( results = [ data.Leaderboard( - id=leaderboard.id, # type: ignore - title=leaderboard.title, # type: ignore - description=leaderboard.description, # type: ignore - resource_id=leaderboard.resource_id, # type: ignore - wallet_connect=leaderboard.wallet_connect, # type: ignore - blockchain_ids=leaderboard.blockchain_ids, # type: ignore - columns_names=leaderboard.columns_names, # type: ignore - created_at=leaderboard.created_at, # type: ignore - updated_at=leaderboard.updated_at, # type: ignore + id=leaderboard.id, + title=leaderboard.title, + description=leaderboard.description, + resource_id=leaderboard.resource_id, + wallet_connect=leaderboard.wallet_connect, + blockchain_ids=leaderboard.blockchain_ids, + columns_names=leaderboard.columns_names, + created_at=leaderboard.created_at, + updated_at=leaderboard.updated_at, ) for leaderboard in leaderboards ] @@ -539,9 +524,7 @@ async def quartiles( @app.get( "/position", - response_model=Union[ - List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition] - ], + response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"], ) async def position( @@ -555,7 +538,7 @@ async def position( ), version: Optional[int] = Query(None, description="Version of the leaderboard."), db_session: Session = Depends(db.yield_db_session), -) -> Union[List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition]]: +) -> List[data.LeaderboardPosition]: """ Returns the leaderboard posotion for the given address. With given window size. @@ -586,36 +569,22 @@ async def position( version, ) - if len(leaderboard.columns_names) > 0: - results = [ - data.LeaderboardUnformattedPosition( - column_1=position.address, - column_2=position.rank, - column_3=position.score, - column_4=position.points_data, - ) - for position in positions - ] - - else: - results = [ - data.LeaderboardPosition( - address=position.address, - score=position.score, - rank=position.rank, - points_data=position.points_data, - ) - for position in positions - ] + results = [ + data.LeaderboardPosition( + address=position.address, + score=position.score, + rank=position.rank, + points_data=position.points_data, + ) + for position in positions + ] return results @app.get( "/rank", - response_model=Union[ - List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition] - ], + response_model=List[data.LeaderboardPosition], tags=["Public Endpoints"], ) async def rank( @@ -625,7 +594,7 @@ async def rank( offset: Optional[int] = Query(None), version: Optional[int] = Query(None, description="Version of the leaderboard."), db_session: Session = Depends(db.yield_db_session), -) -> Union[List[data.LeaderboardPosition], List[data.LeaderboardUnformattedPosition]]: +) -> List[data.LeaderboardPosition]: """ Returns the leaderboard scores for the given rank. """ @@ -651,26 +620,15 @@ async def rank( version_number=version, ) - if len(leaderboard.columns_names) > 0: - results = [ - data.LeaderboardUnformattedPosition( - column_1=position.address, - column_2=position.rank, - column_3=position.score, - column_4=position.points_data, - ) - for position in leaderboard_rank - ] - else: - results = [ - data.LeaderboardPosition( - address=position.address, - score=position.score, - rank=position.rank, - points_data=position.points_data, - ) - for position in leaderboard_rank - ] + results = [ + data.LeaderboardPosition( + address=position.address, + score=position.score, + rank=position.rank, + points_data=position.points_data, + ) + for position in leaderboard_rank + ] return results From 0932c945505d063dc976875dd754278cef303396 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 27 Jan 2024 09:32:12 +0200 Subject: [PATCH 29/35] Remove default names function. --- engineapi/engineapi/actions.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index a1bef295..bc597fbc 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -108,27 +108,6 @@ BATCH_SIGNATURE_PAGE_SIZE = 500 logger = logging.getLogger(__name__) -def get_default_columns_names(columns_names: Optional[Dict[str, str]] = {}): - default_columns_names = {} - - # Retrieve ordered list of field names from LeaderboardPosition - ordered_fields = list(LeaderboardPosition.__annotations__.keys()) - - # Mapping these to the corresponding columns in LeaderboardUnformattedPosition - for index, field_name in enumerate(ordered_fields, start=1): - # Construct the column name like "column_1", "column_2", etc. - column_name = f"column_{index}" - - # If custom column name is provided in columns_names, use it - if columns_names and field_name in columns_names: - default_columns_names[column_name] = columns_names[field_name] - else: - # Otherwise, use the default field name in title case - default_columns_names[column_name] = field_name.replace("_", " ").title() - - return default_columns_names - - def create_dropper_contract( db_session: Session, blockchain: Optional[str], From 243f367d8295ae819ffa22a7e5bc3c46749cc076 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 27 Jan 2024 12:23:14 +0200 Subject: [PATCH 30/35] Add suggested fixes. --- .../versions/71e888082a6d_leaderboard_metadata.py | 9 +++------ engineapi/engineapi/actions.py | 1 - engineapi/engineapi/data.py | 4 ++-- engineapi/engineapi/models.py | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py b/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py index 37ad0604..27b96f56 100644 --- a/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py +++ b/engineapi/alembic/versions/71e888082a6d_leaderboard_metadata.py @@ -1,7 +1,7 @@ """leaderboard metadata Revision ID: 71e888082a6d -Revises: 040f2dfde5a5 +Revises: cc80e886e153 Create Date: 2023-11-15 13:21:16.108399 """ @@ -11,7 +11,7 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "71e888082a6d" -down_revision = "040f2dfde5a5" +down_revision = "cc80e886e153" branch_labels = None depends_on = None @@ -36,10 +36,7 @@ def upgrade(): op.add_column( "leaderboards", sa.Column( - "columns_names", - postgresql.JSONB(astext_type=sa.Text()), - nullable=True, - server_default="{}", + "columns_names", postgresql.JSONB(astext_type=sa.Text()), nullable=True ), ) # ### end Alembic commands ### diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index bc597fbc..8df73261 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1573,7 +1573,6 @@ def update_leaderboard( wallet_connect: Optional[bool], blockchain_ids: Optional[List[int]], columns_names: Optional[ColumnsNames], - delete_names, ) -> Leaderboard: """ Update a leaderboard diff --git a/engineapi/engineapi/data.py b/engineapi/engineapi/data.py index 99fadbec..9830427e 100644 --- a/engineapi/engineapi/data.py +++ b/engineapi/engineapi/data.py @@ -370,7 +370,7 @@ class ColumnsNames(BaseModel): address: Optional[str] = None score: Optional[str] = None points_data: Optional[str] = None - points_data_fields: Dict[str, str] = Field(default_factory=dict) + points_data_fields: Optional[Dict[str, str]] = None class Leaderboard(BaseModel): @@ -380,7 +380,7 @@ class Leaderboard(BaseModel): resource_id: Optional[UUID] = None wallet_connect: bool = False blockchain_ids: List[int] = Field(default_factory=list) - columns_names: ColumnsNames = Field(default_factory=ColumnsNames) + columns_names: Optional[ColumnsNames] = None created_at: datetime updated_at: datetime diff --git a/engineapi/engineapi/models.py b/engineapi/engineapi/models.py index febb5679..9e384f27 100644 --- a/engineapi/engineapi/models.py +++ b/engineapi/engineapi/models.py @@ -350,7 +350,7 @@ class Leaderboard(Base): # type: ignore blockchain_ids = Column(ARRAY(Integer), nullable=False, default=[]) wallet_connect = Column(Boolean, default=False, nullable=False) - columns_names = Column(JSONB, nullable=True, default={}) + columns_names = Column(JSONB, nullable=True) created_at = Column( DateTime(timezone=True), server_default=utcnow(), nullable=False ) From 32b0088fdb5cc5831cfe4c35e44a2342d5ff65ac Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 29 Jan 2024 09:43:03 +0000 Subject: [PATCH 31/35] Bumped bugout package version --- engineapi/engineapi/actions.py | 3 ++- engineapi/requirements.txt | 2 +- engineapi/setup.py | 2 +- moonstreamapi/requirements.txt | 2 +- moonstreamapi/setup.py | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 38537ab3..7852a6ce 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1732,7 +1732,8 @@ def assign_resource( leaderboard.resource_id = resource_id else: resource = create_leaderboard_resource( - leaderboard_id=str(leaderboard_id), user_id=user.id + leaderboard_id=str(leaderboard_id), + user_id=user.id if user is not None else None, ) leaderboard.resource_id = resource.id diff --git a/engineapi/requirements.txt b/engineapi/requirements.txt index 50a372ac..f316bfe1 100644 --- a/engineapi/requirements.txt +++ b/engineapi/requirements.txt @@ -7,7 +7,7 @@ base58==2.1.1 bitarray==2.7.6 boto3==1.27.0 botocore==1.30.0 -bugout==0.2.14 +bugout==0.2.15 certifi==2023.5.7 charset-normalizer==3.1.0 click==8.1.3 diff --git a/engineapi/setup.py b/engineapi/setup.py index d9f4e045..0d24282e 100644 --- a/engineapi/setup.py +++ b/engineapi/setup.py @@ -13,7 +13,7 @@ setup( packages=find_packages(), install_requires=[ "boto3", - "bugout>=0.2.14", + "bugout>=0.2.15", "eip712==0.1.0", "eth-typing>=2.3.0", "fastapi", diff --git a/moonstreamapi/requirements.txt b/moonstreamapi/requirements.txt index 62f4c140..cbc41057 100644 --- a/moonstreamapi/requirements.txt +++ b/moonstreamapi/requirements.txt @@ -9,7 +9,7 @@ base58==2.1.1 bitarray==2.6.0 boto3==1.26.5 botocore==1.29.5 -bugout>=0.2.13 +bugout>=0.2.15 certifi==2022.9.24 charset-normalizer==2.1.1 click==8.1.3 diff --git a/moonstreamapi/setup.py b/moonstreamapi/setup.py index 613bda55..40514803 100644 --- a/moonstreamapi/setup.py +++ b/moonstreamapi/setup.py @@ -13,7 +13,7 @@ setup( install_requires=[ "appdirs", "boto3", - "bugout>=0.2.13", + "bugout>=0.2.15", "fastapi", "moonstreamdb>=0.3.5", "humbug", From e723dabaaf3f357b871c4ab8db4182ad4662e1fc Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 27 Jan 2024 22:12:53 +0200 Subject: [PATCH 32/35] Fix permission_list to permissions. --- engineapi/engineapi/actions.py | 2 +- moonstreamapi/moonstreamapi/actions.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engineapi/engineapi/actions.py b/engineapi/engineapi/actions.py index 7852a6ce..1e9bfb26 100644 --- a/engineapi/engineapi/actions.py +++ b/engineapi/engineapi/actions.py @@ -1692,7 +1692,7 @@ def create_leaderboard_resource(leaderboard_id: str, user_id: Optional[str] = No holder_permissions=BugoutResourceHolder( holder_type=HolderType.user, holder_id=user_id, - permission_list=[ + permissions=[ ResourcePermissions.ADMIN, ResourcePermissions.READ, ResourcePermissions.UPDATE, diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index ad9f7c22..5139eaaf 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -850,6 +850,8 @@ def get_list_of_support_interfaces( Returns list of interfaces supported by given address """ + result = {} + try: _, _, is_contract = check_if_smart_contract( blockchain_type=blockchain_type, address=address, user_token=user_token @@ -865,8 +867,6 @@ def get_list_of_support_interfaces( abi=supportsInterface_abi, ) - result = {} - if blockchain_type in multicall_contracts: calls = [] @@ -979,7 +979,7 @@ def create_resource_for_user( holder_permissions=BugoutResourceHolder( holder_type=HolderType.user, holder_id=user_id, - permission_list=[ + permissions=[ ResourcePermissions.ADMIN, ResourcePermissions.READ, ResourcePermissions.UPDATE, From c7c89aaadd26b011fb9b839f39c91bad3adb7fd9 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sat, 27 Jan 2024 22:33:56 +0200 Subject: [PATCH 33/35] Add app timeout. --- moonstreamapi/moonstreamapi/actions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index 5139eaaf..08530c77 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -965,6 +965,7 @@ def create_resource_for_user( token=MOONSTREAM_ADMIN_ACCESS_TOKEN, application_id=MOONSTREAM_APPLICATION_ID, resource_data=resource_data, + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) except BugoutResponseException as e: raise MoonstreamHTTPException(status_code=e.status_code, detail=e.detail) @@ -986,6 +987,7 @@ def create_resource_for_user( ResourcePermissions.DELETE, ], ), + timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS, ) except BugoutResponseException as e: logger.error( From 346bcae89e6f159fc79fb9c6b439005de6388285 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 29 Jan 2024 06:34:51 +0200 Subject: [PATCH 34/35] Fix old logic code. --- engineapi/engineapi/routes/leaderboard.py | 29 ++++------------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/engineapi/engineapi/routes/leaderboard.py b/engineapi/engineapi/routes/leaderboard.py index 9861044c..8a69e8f8 100644 --- a/engineapi/engineapi/routes/leaderboard.py +++ b/engineapi/engineapi/routes/leaderboard.py @@ -494,30 +494,11 @@ async def quartiles( logger.error(f"Error while getting quartiles: {e}") raise EngineHTTPException(status_code=500, detail="Internal server error") - if len(leaderboard.columns_names) > 0: - result = data.QuartilesResponse( - percentile_25={ - "column_1": q1.address, - "column_2": q1.rank, - "column_3": q1.score, - }, - percentile_50={ - "column_1": q2.address, - "column_2": q2.rank, - "column_3": q2.score, - }, - percentile_75={ - "column_1": q3.address, - "column_2": q3.rank, - "column_3": q3.score, - }, - ) - else: - result = data.QuartilesResponse( - percentile_25={"address": q1.address, "rank": q1.rank, "score": q1.score}, - percentile_50={"address": q2.address, "rank": q2.rank, "score": q2.score}, - percentile_75={"address": q3.address, "rank": q3.rank, "score": q3.score}, - ) + result = data.QuartilesResponse( + percentile_25={"address": q1.address, "rank": q1.rank, "score": q1.score}, + percentile_50={"address": q2.address, "rank": q2.rank, "score": q2.score}, + percentile_75={"address": q3.address, "rank": q3.rank, "score": q3.score}, + ) return result From 0d36368dd2521ec6c9efb3b2516cc4408feb7bcd Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 1 Feb 2024 10:15:47 +0000 Subject: [PATCH 35/35] Updated revision version number --- engineapi/alembic/versions/6d07739cb13e_live_at_for_metatx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engineapi/alembic/versions/6d07739cb13e_live_at_for_metatx.py b/engineapi/alembic/versions/6d07739cb13e_live_at_for_metatx.py index 5db963d4..498705a4 100644 --- a/engineapi/alembic/versions/6d07739cb13e_live_at_for_metatx.py +++ b/engineapi/alembic/versions/6d07739cb13e_live_at_for_metatx.py @@ -1,7 +1,7 @@ """Live at for metatx Revision ID: 6d07739cb13e -Revises: cc80e886e153 +Revises: 71e888082a6d Create Date: 2023-12-06 14:33:04.814144 """ @@ -11,7 +11,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. revision = '6d07739cb13e' -down_revision = 'cc80e886e153' +down_revision = '71e888082a6d' branch_labels = None depends_on = None