Initial version of eth address model with labels

crawlers/moonstreamcrawlers/ethcrawler.py

@@ -171,16 +171,8 @@ def main() -> None:
     parser.set_defaults(func=lambda _: parser.print_help())
     subcommands = parser.add_subparsers(description="Crawlers commands")
 
-    parser_ethcrawler = subcommands.add_parser(
-        "ethcrawler", description="Ethereum crawler"
-    )
-    parser_ethcrawler.set_defaults(func=lambda _: parser_ethcrawler.print_help())
-    subcommands_ethcrawler = parser_ethcrawler.add_subparsers(
-        description="Ethereum crawler commands"
-    )
-
     # Ethereum blocks parser
-    parser_ethcrawler_blocks = subcommands_ethcrawler.add_parser(
+    parser_ethcrawler_blocks = subcommands.add_parser(
         "blocks", description="Ethereum blocks commands"
     )
     parser_ethcrawler_blocks.set_defaults(
@@ -284,7 +276,7 @@ def main() -> None:
         func=ethcrawler_blocks_missing_handler
     )
 
-    parser_ethcrawler_contracts = subcommands_ethcrawler.add_parser(
+    parser_ethcrawler_contracts = subcommands.add_parser(
         "contracts", description="Ethereum smart contract related crawlers"
     )
     parser_ethcrawler_contracts.set_defaults(

crawlers/moonstreamcrawlers/identity.py

@@ -0,0 +1,107 @@
+import argparse
+import json
+import os
+import time
+
+import requests
+
+from moonstreamdb.db import yield_db_session_ctx
+from moonstreamdb.models import EthereumAddress
+
+COINMARKETCAP_API_KEY = os.environ.get("COINMARKETCAP_API_KEY")
+if COINMARKETCAP_API_KEY is None:
+    raise ValueError("COINMARKETCAP_API_KEY environment variable must be set")
+
+CRAWL_ORIGINS = {
+    "pro": "https://pro-api.coinmarketcap.com",
+    "sandbox": "https://sandbox-api.coinmarketcap.com",
+}
+
+
+def identities_cmc_handler(args: argparse.Namespace) -> None:
+    """
+    Parse metadata for Ethereum tokens.
+    """
+    headers = {
+        "X-CMC_PRO_API_KEY": COINMARKETCAP_API_KEY,
+        "Accept": "application/json",
+        "Accept-Encoding": "deflate, gzip",
+    }
+    if args.sandbox:
+        CRAWL_ORIGIN = CRAWL_ORIGINS["sandbox"]
+    else:
+        CRAWL_ORIGIN = CRAWL_ORIGINS["pro"]
+    url = f"{CRAWL_ORIGIN}/v1/cryptocurrency/map"
+
+    start_n = 1
+    limit_n = 5000
+
+    while True:
+        params = {"start": start_n, "limit": limit_n}
+        try:
+            r = requests.get(url=url, headers=headers, params=params)
+            r.raise_for_status()
+            response = r.json()
+        except Exception as err:
+            raise Exception(err)
+
+        if len(response["data"]) == 0:
+            print("No more data, crawling finished")
+            break
+
+        with yield_db_session_ctx() as db_session:
+            for crypto in response["data"]:
+                if crypto["platform"] is not None:
+                    if (
+                        crypto["platform"]["id"] == 1027
+                        and crypto["platform"]["token_address"] is not None
+                    ):
+
+                        eth_token = EthereumAddress(
+                            address=crypto["platform"]["token_address"],
+                            name=crypto["name"],
+                            symbol=crypto["symbol"],
+                        )
+                        db_session.add(eth_token)
+                        print(f"Added {crypto['name']} token")
+
+            db_session.commit()
+        start_n += limit_n
+
+        print(f"Loop ended, starting new from {start_n} to {start_n + limit_n - 1}")
+        time.sleep(1)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Crawls address identities CLI")
+    parser.set_defaults(func=lambda _: parser.print_help())
+    subcommands = parser.add_subparsers(description="Crawlers commands")
+
+    parser_cmc = subcommands.add_parser(
+        "cmc", description="Coinmarketcap commands"
+    )
+    parser_cmc.set_defaults(
+        func=lambda _: parser_cmc.print_help()
+    )
+    subcommands_parser_cmc = parser_cmc.add_subparsers(
+        description="Ethereum blocks commands"
+    )
+    parser_cmc.add_argument(
+        "-s",
+        "--sandbox",
+        action="store_true",
+        help="Target to sandbox API",
+    )
+    parser_cmc.set_defaults(func=identities_cmc_handler)
+
+    parser_label_cloud = subcommands.add_parser(
+        "label_cloud", description="Etherscan label cloud commands"
+    )
+    parser_label_cloud.set_defaults(func=identities_get_handler)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

crawlers/setup.py

@@ -40,6 +40,10 @@ setup(
     ],
     extras_require={"dev": ["black", "mypy", "types-requests"]},
     entry_points={
-        "console_scripts": ["moonstreamcrawlers=moonstreamcrawlers.cli:main"]
+        "console_scripts": [
+            "ethcrawler=moonstreamcrawlers.ethcrawler:main",
+            "esd=moonstreamcrawlers.esd:main",
+            "identity=moonstreamcrawlers.identity:main"
+        ]
     },
 )

db/moonstreamdb/cli.py

@@ -0,0 +1,59 @@
+import argparse
+
+from .db import yield_db_session_ctx
+from .models import EthereumLabel
+
+
+def labels_add_handler(args: argparse.Namespace) -> None:
+    pass
+
+
+def labels_list_handler(args: argparse.Namespace) -> None:
+    """
+    Return list of all labels.
+    """
+    with yield_db_session_ctx() as db_session:
+        labels = db_session.query(EthereumLabel).all()
+
+    print(labels.json())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Crawls address identities CLI")
+    parser.set_defaults(func=lambda _: parser.print_help())
+    subcommands = parser.add_subparsers(description="Crawlers commands")
+
+    parser_labels = subcommands.add_parser("labels", description="Meta labels commands")
+    parser_labels.set_defaults(func=lambda _: parser_labels.print_help())
+    subcommands_labels = parser_labels.add_subparsers(
+        description="Database meta labels commands"
+    )
+
+    parser_labels_add = subcommands_labels.add_parser(
+        "add", description="Add new label command"
+    )
+    parser_labels_add.add_argument(
+        "-a",
+        "--address",
+        required=True,
+        help="Address attach to",
+    )
+    parser_labels_add.add_argument(
+        "-l",
+        "--label",
+        required=True,
+        help="New label name",
+    )
+    parser_labels_add.set_defaults(func=labels_add_handler)
+
+    parser_labels_list = subcommands_labels.add_parser(
+        "list", description="List all meta labels command"
+    )
+    parser_labels_list.set_defaults(func=labels_list_handler)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

db/moonstreamdb/models.py

@@ -1,4 +1,5 @@
-import sqlalchemy
+import uuid
+
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import (
     BigInteger,
@@ -11,6 +12,7 @@ from sqlalchemy import (
     Text,
     VARCHAR,
 )
+from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.sql import expression
 from sqlalchemy.ext.compiler import compiles
 
@@ -100,17 +102,64 @@ class EthereumTransaction(Base):  # type: ignore
     )
 
 
-class EthereumSmartContract(Base):  # type: ignore
-    __tablename__ = "ethereum_smart_contracts"
+class EthereumAddress(Base):  # type: ignore
+    __tablename__ = "ethereum_addresses"
 
-    id = Column(Integer, primary_key=True, autoincrement=True)
+    id = Column(
+        UUID(as_uuid=True),
+        primary_key=True,
+        default=uuid.uuid4,
+        unique=True,
+        nullable=False,
+    )
     transaction_hash = Column(
         VARCHAR(256),
         ForeignKey("ethereum_transactions.hash", ondelete="CASCADE"),
-        nullable=False,
+        nullable=True,
         index=True,
     )
     address = Column(VARCHAR(256), nullable=False, index=True)
+    created_at = Column(
+        DateTime(timezone=True), server_default=utcnow(), nullable=False
+    )
+
+
+class EthereumLabel(Base):  # type: ignore
+    """
+    {
+        label: ERC20,
+        metadata: {
+            name: 123,
+            symbol: qwe
+        }
+    }
+
+    {
+        label: Exchange
+        metadata: {}
+    }
+    """
+
+    __tablename__ = "ethereum_labels"
+
+    id = Column(
+        UUID(as_uuid=True),
+        primary_key=True,
+        default=uuid.uuid4,
+        unique=True,
+        nullable=False,
+    )
+    label = Column(VARCHAR(256), nullable=False, index=True)
+    address = Column(
+        Integer,
+        ForeignKey("ethereum_addresses.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    metadata = Column(JSONB, nullable=True)
+    created_at = Column(
+        DateTime(timezone=True), server_default=utcnow(), nullable=False
+    )
 
 
 class EthereumPendingTransaction(Base):  # type: ignore