Add deletion step.

crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py

@@ -19,6 +19,7 @@ from .db import (
     get_current_metadata_for_address,
     get_tokens_wich_maybe_updated,
     metadata_to_label,
+    clean_labels_from_db,
 )
 from ..settings import (
     MOONSTREAM_STATE_CRAWLER_DB_STATEMENT_TIMEOUT_MILLIS,
@@ -141,6 +142,9 @@ def parse_metadata(
                 for i in range(0, len(tokens_uri_by_address[address]), batch_size)
             ]:
                 writed_labels = 0
+                db_session.commit()
+
+                with db_session.begin():
                     for token_uri_data in requests_chunk:
 
                         if token_uri_data.token_id not in parsed_with_leak:
@@ -156,9 +160,15 @@ def parse_metadata(
                             writed_labels += 1
 
                     if writed_labels > 0:
-                    commit_session(db_session)
+                        clean_labels_from_db(
+                            db_session=db_session,
+                            blockchain_type=blockchain_type,
+                            address=address,
+                        )
                         logger.info(f"Write {writed_labels} labels for {address}")
 
+                # trasaction is commited here
+
     finally:
         db_session.close()
 

crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py

@@ -210,3 +210,43 @@ def get_tokens_wich_maybe_updated(
     result = [data[0] for data in tokens]
 
     return result
+
+
+def clean_labels_from_db(
+    db_session: Session, blockchain_type: AvailableBlockchainType, address: str
+):
+    """
+    Remove existing labels.
+    """
+
+    label_model = get_label_model(blockchain_type)
+
+    table = label_model.__tablename__
+
+    db_session.execute(
+        """ 
+        WITH lates_token_metadata AS (
+            SELECT
+                DISTINCT ON (label_data->>'token_id') label_data->>'token_id' AS token_id,
+                id as id,
+                block_number as block_number
+            FROM
+                {}
+            WHERE
+                label=:label
+                AND address=:address
+            ORDER BY
+                label_data->>'token_id' ASC,
+                block_number DESC
+        )
+        DELETE FROM
+            {} USING lates_token_metadata
+        WHERE
+            label=:label
+            AND address=:address
+            AND polygon_labels.id  not in (select id from lates_token_metadata) RETURNING polygon_labels.block_number;
+    """.format(
+            table, table
+        ),
+        {"address": address, "label": METADATA_CRAWLER_LABEL},
+    )