Merge branch 'main' into query-parameters-endpoint

backend/moonstreamapi/admin/cli.py

@@ -39,6 +39,35 @@ def parse_boolean_arg(raw_arg: Optional[str]) -> Optional[bool]:
     return False
 
 
+def migration_run(step_map, command, step, step_order):
+    if step is None:
+        # run all steps
+
+        if step_order is None:
+            raise ValueError(
+                f"step_order is required when running all steps for {command}"
+            )
+
+        if command == "downgrade":
+            step_order = reversed(step_order)
+
+        for step in step_order:
+            logger.info(
+                f"Starting step {step}: {step_map[command][step]['description']}"
+            )
+            migration_function = step_map[command][step]["action"]
+            if callable(migration_function):
+                migration_function()
+    elif step in step_map[command]:
+        logger.info(f"Starting step {step}: {step_map[command][step]['description']}")
+        migration_function = step_map[command][step]["action"]
+        if callable(migration_function):
+            migration_function()
+    else:
+        logger.error(f"Step {step} not found in {command}")
+        logger.info(f"Available steps: {step_map[command].keys()}")
+
+
 def migrations_list(args: argparse.Namespace) -> None:
     migrations_overview = f"""
 
@@ -55,6 +84,9 @@ description: {generate_entity_subscriptions.__doc__}
 steps:
     - step 1: generate_entity_subscriptions_from_brood_resources - Generate entity subscriptions from brood resources
     - step 2: update_dashboards_connection - Update dashboards connection
+- id: 20230501
+name: fix_duplicates_keys_in_entity_subscription
+description: Fix entity duplicates keys for all subscriptions introduced in 20230213
     """
     logger.info(entity_migration_overview)
 
@@ -85,8 +117,31 @@ def migrations_run(args: argparse.Namespace) -> None:
     web3_session = yield_web3_provider()
     db_session = SessionLocal()
     try:
-        if args.id == 20230213:
+        if args.id == 20230501:
+            # fix entity duplicates keys for all subscriptions introduced in 20230213
+
+            step_order = ["fix_duplicates_keys_in_entity_subscription"]
             step_map: Dict[str, Dict[str, Any]] = {
+                "upgrade": {
+                    "fix_duplicates_keys_in_entity_subscription": {
+                        "action": generate_entity_subscriptions.fix_duplicates_keys_in_entity_subscription,
+                        "description": "Fix entity duplicates keys for all subscriptions introduced in 20230213",
+                    },
+                },
+                "downgrade": {},
+            }
+            if args.command not in ["upgrade", "downgrade"]:
+                logger.info("Wrong command. Please use upgrade or downgrade")
+            step = args.step
+
+            migration_run(step_map, args.command, step, step_order)
+
+        if args.id == 20230213:
+            step_order = [
+                "generate_entity_subscriptions_from_brood_resources",
+                "update_dashboards_connection",
+            ]
+            step_map = {
                 "upgrade": {
                     "generate_entity_subscriptions_from_brood_resources": {
                         "action": generate_entity_subscriptions.generate_entity_subscriptions_from_brood_resources,
@@ -112,26 +167,7 @@ def migrations_run(args: argparse.Namespace) -> None:
                 logger.info("Wrong command. Please use upgrade or downgrade")
             step = args.step
 
-            if step is None:
-                # run all steps
-
-                for step in step_map[args.command]:
-                    logger.info(
-                        f"Starting step {step}: {step_map[args.command][step]['description']}"
-                    )
-                    migration_function = step_map[args.command][step]["action"]
-                    if callable(migration_function):
-                        migration_function()
-            elif step in step_map[args.command]:
-                logger.info(
-                    f"Starting step {step}: {step_map[args.command][step]['description']}"
-                )
-                migration_function = step_map[args.command][step]["action"]
-                if callable(migration_function):
-                    migration_function()
-            else:
-                logger.info(f"Step {step} does not exist")
-                logger.info(f"Available steps: {step_map[args.command].keys()}")
+            migration_run(step_map, args.command, step, step_order)
 
         elif args.id == 20211101:
             logger.info("Starting update of subscriptions in Brood resource...")

backend/moonstreamapi/admin/migrations/generate_entity_subscriptions.py

@@ -716,3 +716,140 @@ def restore_dashboard_state():
                 traceback.print_exc()
                 logger.error(f"Failed to update dashboard: {str(e)}")
                 continue
+
+
+def fix_duplicates_keys_in_entity_subscription():
+    """
+    Migration generate_entity_subscriptions_from_brood_resources
+    create duplicates keys "secondary_fields" subscriptions secondary_fields
+    "secondary_fields": {
+                "secondary_fields": {
+        ...
+        }
+    }
+    That function will remove internal secondary_fields and flat all keys to one level upper
+    """
+
+    # get all entities from subscriptions
+
+    subscriptions: BugoutResources = bc.list_resources(
+        token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+        params={"type": BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION},
+        timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
+    )
+
+    # get collection ids from that resources
+
+    collection_id_user_id_mappig = {}
+
+    for subscription in subscriptions.resources:
+        if "collection_id" in subscription.resource_data:
+            if (
+                subscription.resource_data["collection_id"]
+                not in collection_id_user_id_mappig
+            ):
+                collection_id_user_id_mappig[
+                    subscription.resource_data["collection_id"]
+                ] = subscription.resource_data["user_id"]
+            else:
+                raise Exception(
+                    f"Duplicate collection_id {subscription.resource_data['collection_id']} in subscriptions"
+                )
+    # go through all collections and fix entities.
+    # Will creating one new entity with same data but without "type:subscription" in required_fields
+
+    for collection_id, user_id in collection_id_user_id_mappig.items():
+        # get collection entities
+
+        collection_entities = ec.search_entities(
+            token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+            collection_id=collection_id,
+            required_field=[f"type:subscription"],
+            limit=1000,
+        )
+
+        logger.info(
+            f"Amount of entities in user: {user_id} collection {collection_id}: {len(collection_entities.entities)}"
+        )
+
+        for entity in collection_entities.entities:
+            # get entity data
+
+            if entity.secondary_fields is None:
+                continue
+
+            secondary_fields = entity.secondary_fields
+
+            if "secondary_fields" not in secondary_fields:
+                continue
+
+            secondary_fields = secondary_fields["secondary_fields"]
+
+            # get entity id
+
+            entity_id = entity.entity_id
+
+            # get entity type
+
+            entity_type = None
+
+            # extract required fields
+            for entity_required_field in entity.required_fields:
+                if "type" in entity_required_field:
+                    entity_type = entity_required_field["type"]
+            if entity_type != "subscription":
+                continue
+
+            # Create new entity with same data but without "type:subscription" in required_fields
+
+            try:
+                new_required_fields = [
+                    entity_field
+                    for entity_field in entity.required_fields
+                    if "type" not in entity_field
+                ]
+                new_required_fields.append(
+                    {"type": "copy_of_malformed_entity_20230213"}
+                )
+                new_required_fields.append({"entity_id": str(entity_id)})
+
+                new_entity = ec.add_entity(
+                    token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+                    collection_id=collection_id,
+                    blockchain=entity.blockchain,
+                    address=entity.address,
+                    name=entity.name,
+                    required_fields=new_required_fields,
+                    secondary_fields=entity.secondary_fields,
+                )
+                logger.info(
+                    f"Entity {new_entity.entity_id} created successfully for collection {collection_id}"
+                )
+
+            except Exception as e:
+                logger.error(
+                    f"Failed to create entity {entity_id} for collection {collection_id}: {str(e)}, user_id: {user_id}"
+                )
+                continue
+
+            # Update old entity without secondary_fields duplicate
+
+            try:
+                ec.update_entity(
+                    token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
+                    collection_id=collection_id,
+                    entity_id=entity_id,
+                    blockchain=entity.blockchain,
+                    address=entity.address,
+                    name=entity.name,
+                    required_fields=entity.required_fields,
+                    secondary_fields=secondary_fields,
+                )
+                logger.info(
+                    f"Entity {entity_id} updated successfully for collection {collection_id}"
+                )
+
+            except Exception as e:
+                logger.error(
+                    f"Failed to update entity {entity_id} for collection {collection_id}: {str(e)}, user_id: {user_id}"
+                )

backend/setup.py

@@ -14,7 +14,7 @@ setup(
         "appdirs",
         "boto3",
         "bugout>=0.1.19",
-        "moonstream-entity>=0.0.4",
+        "moonstream-entity>=0.0.5",
         "fastapi",
         "moonstreamdb>=0.3.3",
         "humbug",

crawlers/mooncrawl/setup.py

@@ -39,7 +39,7 @@ setup(
         "fastapi",
         "moonstreamdb>=0.3.3",
         "moonstream>=0.1.1",
-        "moonstream-entity>=0.0.4",
+        "moonstream-entity>=0.0.5",
         "moonworm[moonstream]>=0.6.2",
         "humbug",
         "pydantic==1.9.2",

frontend/src/components/SubscriptionCard.js

@@ -308,21 +308,20 @@ const SubscriptionCard = ({ subscription, isDesktopView, iconLink }) => {
             {subscription.abi ? (
               <CheckIcon />
             ) : (
-              <></>
-              // <Button
-              //   colorScheme="orange"
-              //   size="xs"
-              //   py={2}
-              //   disabled={!subscription.address}
-              //   onClick={() =>
-              //     overlay.toggleModal({
-              //       type: MODAL_TYPES.UPLOAD_ABI,
-              //       props: { id: subscription.id },
-              //     })
-              //   }
-              // >
-              //   Upload
-              // </Button>
+              <Button
+                colorScheme="orange"
+                size="xs"
+                py={2}
+                disabled={!subscription.address}
+                onClick={() =>
+                  overlay.toggleModal({
+                    type: MODAL_TYPES.UPLOAD_ABI,
+                    props: { id: subscription.id },
+                  })
+                }
+              >
+                Upload
+              </Button>
             )}
           </Td>
           <Td {...cellProps}>