Merge branch 'main' into add-batch-jobs-endpoint

.github/workflows/release.moonstreamdb.yml

@@ -7,7 +7,7 @@ on:
 
 defaults:
   run:
-    working-directory: db
+    working-directory: moonstreamdb
 
 jobs:
   publish:

clients/python/README.md

@@ -82,7 +82,7 @@ Stream of event packs will be generating from recent timestamp to older and inne
 
 **From timestamp to timestamp, from bottom to top**
 
-When `start_time` is less then `end_time`.
+When `start_time` is less than `end_time`.
 
 ```python
 for events in mc.create_stream(

crawlers/deploy/deploy.bash

@@ -106,6 +106,11 @@ WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transaction
 WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="wyrm-historical-crawl-events.service"
 WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="wyrm-historical-crawl-events.timer"
 
+# ZkSync Era testnet
+ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service"
+ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE="zksync-era-testnet-missing.service"
+ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE="zksync-era-testnet-missing.service"
+
 set -eu
 
 echo
@@ -499,3 +504,21 @@ cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/home/ubuntu/.c
 cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
+
+# ZkSync Era
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet block with transactions syncronizer service definition with ${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet missing service and timer with: ${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"

crawlers/deploy/zksync-era-testnet-missing.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Fill missing blocks at ZkSync Era testnet database
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era_testnet -n
+CPUWeight=50
+SyslogIdentifier=zksync-era-testnet-missing

crawlers/deploy/zksync-era-testnet-missing.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Fill missing blocks at ZkSync Era testnet database
+
+[Timer]
+OnBootSec=120s
+OnUnitActiveSec=15m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/zksync-era-testnet-synchronize.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=ZkSync Era testnet block with transactions synchronizer
+StartLimitIntervalSec=300
+StartLimitBurst=3
+After=network.target
+
+[Service]
+Restart=on-failure
+RestartSec=15s
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2
+CPUWeight=90
+SyslogIdentifier=zksync-era-testnet-synchronize
+
+[Install]
+WantedBy=multi-user.target

crawlers/mooncrawl/mooncrawl/blockchain.py

@@ -27,6 +27,7 @@ from .settings import (
     MOONSTREAM_POLYGON_WEB3_PROVIDER_URI,
     MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
     MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
+    MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI,
     NB_ACCESS_ID_HEADER,
     NB_DATA_SOURCE_HEADER,
     WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
@@ -70,6 +71,8 @@ def connect(
             web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI
         elif blockchain_type == AvailableBlockchainType.WYRM:
             web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
+        elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+            web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
         else:
             raise Exception("Wrong blockchain type provided for web3 URI")
 
@@ -123,6 +126,19 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
     )
     if blockchain_type == AvailableBlockchainType.XDAI:
         block_obj.author = block.author
+    if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+        block_obj.mix_hash = block.get("mixHash", "")
+        block_obj.sha3_uncles = block.get("sha3Uncles", "")
+        block_obj.l1_batch_number = (
+            int(block.get("l1BatchNumber"), 0)
+            if block.get("l1BatchNumber") is not None
+            else None
+        )
+        block_obj.l1_batch_timestamp = (
+            int(block.get("l1BatchTimestamp"), 0)
+            if block.get("l1BatchTimestamp") is not None
+            else None
+        )
 
     db_session.add(block_obj)
 
@@ -152,6 +168,17 @@ def add_block_transactions(
             transaction_type=int(tx["type"], 0) if tx.get("type") is not None else None,
             value=tx.value,
         )
+        if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+            tx_obj.l1_batch_number = (
+                int(tx.get("l1BatchNumber"), 0)
+                if tx.get("l1BatchNumber") is not None
+                else None
+            )
+            tx_obj.l1_batch_tx_index = (
+                int(tx.get("l1BatchTxIndex"), 0)
+                if tx.get("l1BatchTxIndex") is not None
+                else None
+            )
 
         db_session.add(tx_obj)
 

crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py

@@ -130,6 +130,8 @@ def continuous_crawler(
         network = Network.xdai
     elif blockchain_type == AvailableBlockchainType.WYRM:
         network = Network.wyrm
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+        network = Network.zksync_era_testnet
     else:
         raise ValueError(f"Unknown blockchain type: {blockchain_type}")
 

crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py

@@ -35,6 +35,7 @@ class SubscriptionTypes(Enum):
     MUMBAI_BLOCKCHAIN = "mumbai_smartcontract"
     XDAI_BLOCKCHAIN = "xdai_smartcontract"
     WYRM_BLOCKCHAIN = "wyrm_smartcontract"
+    ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
 
 
 def abi_input_signature(input_abi: Dict[str, Any]) -> str:
@@ -139,6 +140,8 @@ def blockchain_type_to_subscription_type(
         return SubscriptionTypes.XDAI_BLOCKCHAIN
     elif blockchain_type == AvailableBlockchainType.WYRM:
         return SubscriptionTypes.WYRM_BLOCKCHAIN
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+        return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
     else:
         raise ValueError(f"Unknown blockchain type: {blockchain_type}")
 

crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py

@@ -68,6 +68,8 @@ def function_call_crawler(
         network = Network.xdai
     elif blockchain_type == AvailableBlockchainType.WYRM:
         network = Network.wyrm
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+        network = Network.zksync_era_testnet
     else:
         raise ValueError(f"Unknown blockchain type: {blockchain_type}")
 

crawlers/mooncrawl/mooncrawl/settings.py

@@ -114,6 +114,14 @@ MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get(
 if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "":
     raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set")
 
+MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI = os.environ.get(
+    "MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI", ""
+)
+if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "":
+    raise Exception(
+        "MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set"
+    )
+
 MOONSTREAM_CRAWL_WORKERS = 4
 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
 try:

crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py

@@ -48,6 +48,7 @@ subscription_id_by_blockchain = {
     "mumbai": "mumbai_smartcontract",
     "xdai": "xdai_smartcontract",
     "wyrm": "wyrm_smartcontract",
+    "zksync_era_testnet": "zksync_era_testnet_smartcontract",
 }
 
 blockchain_by_subscription_id = {
@@ -56,11 +57,13 @@ blockchain_by_subscription_id = {
     "mumbai_blockchain": "mumbai",
     "xdai_blockchain": "xdai",
     "wyrm_blockchain": "wyrm",
+    "zksync_era_testnet_blockchain": "zksync_era_testnet",
     "ethereum_smartcontract": "ethereum",
     "polygon_smartcontract": "polygon",
     "mumbai_smartcontract": "mumbai",
     "xdai_smartcontract": "xdai",
     "wyrm_smartcontract": "wyrm",
+    "zksync_era_testnet_smartcontract": "zksync_era_testnet",
 }
 
 

crawlers/mooncrawl/mooncrawl/version.py

@@ -2,4 +2,4 @@
 Moonstream crawlers version.
 """
 
-MOONCRAWL_VERSION = "0.3.1"
+MOONCRAWL_VERSION = "0.3.2"

crawlers/mooncrawl/sample.env

@@ -25,6 +25,7 @@ export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://<connection_path_uri_to_pol
 export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_mumbai_node>"
 export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
 export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_node>"
+export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_testnet_node>"
 export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
 
 # AWS environment variables

crawlers/mooncrawl/setup.py

@@ -37,7 +37,7 @@ setup(
         "bugout>=0.2.8",
         "chardet",
         "fastapi",
-        "moonstreamdb>=0.3.3",
+        "moonstreamdb>=0.3.4",
         "moonstream>=0.1.1",
         "moonstream-entity>=0.0.5",
         "moonworm[moonstream]>=0.6.2",

docs/architecture.md

@@ -49,7 +49,7 @@ the schema for this Postgres database as well as migrations that you can use to
 database yourself.
 
 The [`db/`](../db/) directory contains:
-1. A Python package called `moonstreamdb` which defines the databse schema and can be used as a
+1. A Python package called `moonstreamdb` which defines the database schema and can be used as a
 Python library to interact with the data store.
 2. [Alembic](https://alembic.sqlalchemy.org/en/latest/) migrations which can be used via the
 [`alembic.sh`](../db/alembic.sh) shell script to run the migrations against a Postgres database

moonstreamapi/moonstreamapi/actions.py

@@ -59,11 +59,13 @@ blockchain_by_subscription_id = {
     "mumbai_blockchain": "mumbai",
     "xdai_blockchain": "xdai",
     "wyrm_blockchain": "wyrm",
+    "zksync_era_testnet_blockchain": "zksync_era_testnet",
     "ethereum_smartcontract": "ethereum",
     "polygon_smartcontract": "polygon",
     "mumbai_smartcontract": "mumbai",
     "xdai_smartcontract": "xdai",
     "wyrm_smartcontract": "wyrm",
+    "zksync_era_testnet_smartcontract": "zksync_era_testnet",
 }
 
 

moonstreamapi/moonstreamapi/admin/cli.py

@@ -478,18 +478,12 @@ This CLI is configured to work with the following API URLs:
 
     parser_moonworm_tasks_add.set_defaults(func=moonworm_tasks_add_subscription_handler)
 
-
-
     queries_parser = subcommands.add_parser(
         "queries", description="Manage Moonstream queries"
     )
     queries_parser.set_defaults(func=lambda _: queries_parser.print_help())
 
-
+    queries_subcommands = queries_parser.add_subparsers(description="Query commands")
-    
-    queries_subcommands = queries_parser.add_subparsers(
-        description="Query commands"
-    )
 
     create_query_parser = queries_subcommands.add_parser(
         "create-template", description="Create query template"

moonstreamapi/moonstreamapi/admin/queries.py

@@ -24,12 +24,11 @@ from ..actions import get_all_entries_from_search, name_normalization
 logger = logging.getLogger(__name__)
 
 
-
 def create_query_template(args: argparse.Namespace) -> None:
     """
     Create query template for all queries resources.
     """
-    
+
     query = ""
     with args.query_file:
         query = textwrap.indent(args.query_file.read(), "    ")
@@ -39,7 +38,6 @@ def create_query_template(args: argparse.Namespace) -> None:
     name = f"template_{name_normalization(args.name)}"
 
     try:
-
         entry = bc.create_entry(
             journal_id=MOONSTREAM_QUERIES_JOURNAL_ID,
             title=args.name,
@@ -51,18 +49,17 @@ def create_query_template(args: argparse.Namespace) -> None:
             token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
             timeout=BUGOUT_REQUEST_TIMEOUT_SECONDS,
         )
-    
+
     except BugoutResponseException as err:
         logger.error(f"Failed to create query template: {err}")
         return
     except Exception as err:
         logger.error(f"Failed to create query template: {err}")
         return
-    
+
     logger.info(f"Query template created: {entry.id}")
     logger.info(f"Query template created url name: {name}")
 
-
     ### Add query id
 
     try:
@@ -81,4 +78,4 @@ def create_query_template(args: argparse.Namespace) -> None:
         logger.error(f"Failed to add query id: {err}")
         return
 
-    logger.info(f"Query created: {json.dumps(entry.dict(), indent=4)}")
+    logger.info(f"Query created: {json.dumps(entry.dict(), indent=4)}")

moonstreamapi/moonstreamapi/admin/subscription_types.py

@@ -72,6 +72,17 @@ CANONICAL_SUBSCRIPTION_TYPES = {
         stripe_price_id=None,
         active=True,
     ),
+    "zksync_era_testnet_smartcontract": SubscriptionTypeResourceData(
+        id="zksync_era_testnet_smartcontract",
+        name="zkSync Era testnet smartcontract",
+        blockchain="zksync_era_testnet",
+        choices=["input:address", "tag:erc721"],
+        description="Contracts events and tx_calls of contract of zkSync Era testnet blockchain.",
+        icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/zksync-era-testnet-token-logo.png",
+        stripe_product_id=None,
+        stripe_price_id=None,
+        active=True,
+    ),
     "ethereum_blockchain": SubscriptionTypeResourceData(
         id="ethereum_blockchain",
         name="Ethereum transactions",
@@ -127,6 +138,17 @@ CANONICAL_SUBSCRIPTION_TYPES = {
         stripe_price_id=None,
         active=False,
     ),
+    "zksync_era_testnet_blockchain": SubscriptionTypeResourceData(
+        id="zksync_era_testnet_blockchain",
+        name="zkSync Era testnet transactions",
+        blockchain="zksync_era_testnet",
+        choices=["input:address", "tag:erc721"],
+        description="ZkSync Era testnet chain transactions subscription.",
+        icon_url="https://s3.amazonaws.com/static.simiotics.com/moonstream/assets/zksync-era-testnet-token-logo.png",
+        stripe_product_id=None,
+        stripe_price_id=None,
+        active=False,
+    ),
     "ethereum_whalewatch": SubscriptionTypeResourceData(
         id="ethereum_whalewatch",
         name="Ethereum whale watch",

moonstreamapi/moonstreamapi/data.py

@@ -2,14 +2,17 @@
 Pydantic schemas for the Moonstream HTTP API
 """
 from datetime import datetime
+import json
 from enum import Enum
 from typing import Any, Dict, List, Optional, Union, Literal
 from uuid import UUID
 from xmlrpc.client import Boolean
 
+from fastapi import Form
 from pydantic import BaseModel, Field, validator
 from sqlalchemy import false
 
+
 USER_ONBOARDING_STATE = "onboarding_state"
 
 BUGOUT_RESOURCE_QUERY_RESOLVER = "query_name_resolver"
@@ -47,6 +50,8 @@ class SubscriptionResourceData(BaseModel):
     abi: Optional[str]
     color: Optional[str]
     label: Optional[str]
+    description: Optional[str] = None
+    tags: List[str] = Field(default_factory=list)
     user_id: str
     subscription_type_id: Optional[str]
     jobs_status: Optional[List[Dict[str, Any]]] = Field(default_factory=list)
@@ -244,6 +249,40 @@ class SubdcriptionsAbiResponse(BaseModel):
     abi: str
 
 
+class UpdateSubscriptionRequest(BaseModel):
+    color: Optional[str] = Form(None)
+    label: Optional[str] = Form(None)
+    abi: Optional[str] = Form(None)
+    description: Optional[str] = Form(None)
+    tags: Optional[List[Dict[str, str]]] = Form(None)
+
+    @validator("tags", pre=True, always=True)
+    def transform_to_dict(cls, v):
+        if isinstance(v, str):
+            return json.loads(v)
+        elif isinstance(v, list):
+            return v
+        return []
+
+
+class CreateSubscriptionRequest(BaseModel):
+    address: str = Form(...)
+    subscription_type_id: str = Form(...)
+    color: str = Form(...)
+    label: str = Form(...)
+    abi: Optional[str] = Form(None)
+    description: Optional[str] = Form(None)
+    tags: Optional[List[Dict[str, str]]] = Form(None)
+
+    @validator("tags", pre=True, always=True)
+    def transform_to_dict(cls, v):
+        if isinstance(v, str):
+            return json.loads(v)
+        elif isinstance(v, list):
+            return v
+        return []
+
+
 class DashboardMeta(BaseModel):
     subscription_id: UUID
     generic: Optional[List[Dict[str, str]]]
@@ -296,8 +335,8 @@ class QueryInfoResponse(BaseModel):
     preapprove: bool = False
     approved: bool = False
     parameters: Dict[str, Any] = Field(default_factory=dict)
-    created_at: Optional[datetime]
+    created_at: Optional[datetime] = None
-    updated_at: Optional[datetime]
+    updated_at: Optional[datetime] = None
 
 
 class SuggestedQueriesResponse(BaseModel):

moonstreamapi/moonstreamapi/providers/__init__.py

@@ -51,10 +51,12 @@ event_providers: Dict[str, Any] = {
     moonworm_provider.PolygonMoonwormProvider.event_type: moonworm_provider.PolygonMoonwormProvider,
     moonworm_provider.MumbaiMoonwormProvider.event_type: moonworm_provider.MumbaiMoonwormProvider,
     moonworm_provider.XDaiMoonwormProvider.event_type: moonworm_provider.XDaiMoonwormProvider,
+    moonworm_provider.ZkSyncEraTestnetMoonwormProvider.event_type: moonworm_provider.ZkSyncEraTestnetMoonwormProvider,
     transactions.EthereumTransactions.event_type: transactions.EthereumTransactions,
     transactions.PolygonTransactions.event_type: transactions.PolygonTransactions,
     transactions.MumbaiTransactions.event_type: transactions.MumbaiTransactions,
     transactions.XDaiTransactions.event_type: transactions.XDaiTransactions,
+    transactions.ZkSyncEraTestnetTransactions.event_type: transactions.ZkSyncEraTestnetTransactions,
     bugout.polygon_whalewatch_provider.event_type: bugout.polygon_whalewatch_provider,
     bugout.ethereum_txpool_provider.event_type: bugout.ethereum_txpool_provider,
     bugout.ethereum_whalewatch_provider.event_type: bugout.ethereum_whalewatch_provider,

moonstreamapi/moonstreamapi/providers/moonworm_provider.py

@@ -21,6 +21,7 @@ ethereum_event_type = "ethereum_blockchain"
 polygon_event_type = "polygon_blockchain"
 mumbai_event_type = "mumbai_blockchain"
 xdai_event_type = "xdai_blockchain"
+zksync_era_testnet_event_type = "zksync_era_testnet_blockchain"
 allowed_tags = ["tag:erc721"]
 
 description = f"""Event provider for transactions from the Ethereum blockchain.
@@ -413,3 +414,10 @@ XDaiMoonwormProvider = MoonwormProvider(
     description="Provider for reviving transactions from XDai tables.",
     streamboaundary_range_limit=2 * 60 * 60,
 )
+
+ZkSyncEraTestnetMoonwormProvider = MoonwormProvider(
+    event_type="zksync_era_testnet_smartcontract",
+    blockchain=AvailableBlockchainType("zksync_era_testnet"),
+    description="Provider for reviving transactions from zkSync Era testnet tables.",
+    streamboaundary_range_limit=2 * 60 * 60,
+)

moonstreamapi/moonstreamapi/providers/transactions.py

@@ -475,3 +475,10 @@ XDaiTransactions = TransactionsProvider(
     description="Provider for resiving transactions from XDai tables.",
     streamboaundary_range_limit=2 * 60 * 60,
 )
+
+ZkSyncEraTestnetTransactions = TransactionsProvider(
+    event_type="zksync_era_testnet_blockchain",
+    blockchain=AvailableBlockchainType("zksync_era_testnet"),
+    description="Provider for resiving transactions from ZkSync Era testnet tables.",
+    streamboaundary_range_limit=2 * 60 * 60,
+)

moonstreamapi/moonstreamapi/routes/queries.py

@@ -7,7 +7,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 from uuid import UUID
 
 
-
 from bugout.data import BugoutResources, BugoutJournalEntryContent, BugoutJournalEntry
 from bugout.exceptions import BugoutResponseException
 from fastapi import APIRouter, Body, Request
@@ -157,7 +156,6 @@ async def create_query_handler(
     return entry
 
 
-
 @router.get("/templates", tags=["queries"])
 def get_suggested_queries(
     supported_interfaces: Optional[List[str]] = None,
@@ -223,7 +221,6 @@ async def get_query_handler(
 ) -> data.QueryInfoResponse:
     token = request.state.token
 
-
     # normalize query name
 
     try:
@@ -234,7 +231,6 @@ async def get_query_handler(
             detail=f"Provided query name can't be normalize please select different.",
         )
 
-
     # check in templates
 
     try:
@@ -283,7 +279,6 @@ async def get_query_handler(
     else:
         query_id = entries.results[0].entry_url.split("/")[-1]
 
-
     entry = entries.results[0]
 
     try:
@@ -312,7 +307,6 @@ async def get_query_handler(
         else:
             query_parameters[param] = None
 
-
     print(type(entry.created_at))
 
     return data.QueryInfoResponse(
@@ -395,7 +389,6 @@ async def update_query_data_handler(
             detail=f"Provided query name can't be normalize please select different.",
         )
 
-
     # check in templates
 
     try:
@@ -510,9 +503,7 @@ async def get_access_link_handler(
             token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
             journal_id=MOONSTREAM_QUERIES_JOURNAL_ID,
             query=f"tag:query_template tag:query_url:{query_name_normalized}",
-            filters=[
+            filters=[f"context_type:{MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE}"],
-                f"context_type:{MOONSTREAM_QUERY_TEMPLATE_CONTEXT_TYPE}"
-            ],
             limit=1,
         )
     except BugoutResponseException as e:
@@ -522,7 +513,6 @@ async def get_access_link_handler(
         raise MoonstreamHTTPException(status_code=500, internal_error=e)
 
     if len(entries.results) == 0:
-
         try:
             query_id = get_query_by_name(query_name, token)
         except NameNormalizationException:
@@ -532,7 +522,6 @@ async def get_access_link_handler(
             )
 
         try:
-
             entries = bc.search(
                 token=MOONSTREAM_ADMIN_ACCESS_TOKEN,
                 journal_id=MOONSTREAM_QUERIES_JOURNAL_ID,
@@ -552,7 +541,6 @@ async def get_access_link_handler(
             )
 
     try:
-
         s3_response = None
 
         if entries.results[0].content:
@@ -631,4 +619,4 @@ async def remove_query_handler(
     except Exception as e:
         raise MoonstreamHTTPException(status_code=500, internal_error=e)
 
-    return entry
+    return entry

moonstreamapi/moonstreamapi/routes/subscriptions.py

@@ -6,7 +6,6 @@ import hashlib
 import json
 import logging
 from typing import Any, Dict, List, Optional
-import traceback
 
 from bugout.exceptions import BugoutResponseException
 from bugout.data import BugoutSearchResult
@@ -29,7 +28,11 @@ from ..admin import subscription_types
 from ..middleware import MoonstreamHTTPException
 from ..reporter import reporter
 from ..settings import bugout_client as bc, entity_client as ec
-from ..settings import MOONSTREAM_ADMIN_ACCESS_TOKEN, THREAD_TIMEOUT_SECONDS
+from ..settings import (
+    MOONSTREAM_ADMIN_ACCESS_TOKEN,
+    MOONSTREAM_ENTITIES_RESERVED_TAGS,
+    THREAD_TIMEOUT_SECONDS,
+)
 from ..web3_provider import (
     yield_web3_provider,
 )
@@ -49,11 +52,6 @@ BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION = "entity_subscription"
 async def add_subscription_handler(
     request: Request,
     background_tasks: BackgroundTasks,
-    address: str = Form(...),
-    color: str = Form(...),
-    label: str = Form(...),
-    subscription_type_id: str = Form(...),
-    abi: Optional[str] = Form(None),
     web3: Web3 = Depends(yield_web3_provider),
 ) -> data.SubscriptionResourceData:
     """
@@ -61,6 +59,21 @@ async def add_subscription_handler(
     """
     token = request.state.token
 
+    form = await request.form()
+
+    try:
+        form_data = data.CreateSubscriptionRequest(**form)
+    except Exception as e:
+        raise MoonstreamHTTPException(status_code=400, detail=str(e))
+
+    address = form_data.address
+    color = form_data.color
+    label = form_data.label
+    abi = form_data.abi
+    description = form_data.description
+    tags = form_data.tags
+    subscription_type_id = form_data.subscription_type_id
+
     if subscription_type_id != "ethereum_whalewatch":
         try:
             address = web3.toChecksumAddress(address)
@@ -124,6 +137,28 @@ async def add_subscription_handler(
             address,
         )
 
+    if description:
+        content["description"] = description
+
+    allowed_required_fields = []
+    if tags:
+        allowed_required_fields = [
+            item
+            for item in tags
+            if not any(key in item for key in MOONSTREAM_ENTITIES_RESERVED_TAGS)
+        ]
+
+    required_fields = [
+        {"type": "subscription"},
+        {"subscription_type_id": f"{subscription_type_id}"},
+        {"color": f"{color}"},
+        {"label": f"{label}"},
+        {"user_id": f"{user.id}"},
+    ]
+
+    if allowed_required_fields:
+        required_fields.extend(allowed_required_fields)
+
     try:
         collection_id = get_entity_subscription_collection_id(
             resource_type=BUGOUT_RESOURCE_TYPE_ENTITY_SUBSCRIPTION,
@@ -140,13 +175,7 @@ async def add_subscription_handler(
                 subscription_type_id
             ].blockchain,
             name=label,
-            required_fields=[
+            required_fields=required_fields,
-                {"type": "subscription"},
-                {"subscription_type_id": f"{subscription_type_id}"},
-                {"color": f"{color}"},
-                {"label": f"{label}"},
-                {"user_id": f"{user.id}"},
-            ],
             secondary_fields=content,
         )
     except EntityCollectionNotFoundException as e:
@@ -163,6 +192,13 @@ async def add_subscription_handler(
             detail="Currently unable to get collection id",
         )
 
+    normalized_entity_tags = [
+        f"{key}:{value}"
+        for tag in entity.required_fields
+        for key, value in tag.items()
+        if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS
+    ]
+
     return data.SubscriptionResourceData(
         id=str(entity.entity_id),
         user_id=str(user.id),
@@ -170,6 +206,8 @@ async def add_subscription_handler(
         color=color,
         label=label,
         abi=entity.secondary_fields.get("abi"),
+        description=entity.secondary_fields.get("description"),
+        tags=normalized_entity_tags,
         subscription_type_id=subscription_type_id,
         updated_at=entity.updated_at,
         created_at=entity.created_at,
@@ -240,6 +278,8 @@ async def delete_subscription_handler(request: Request, subscription_id: str):
         color=color,
         label=label,
         abi=abi,
+        description=deleted_entity.secondary_fields.get("description"),
+        tags=deleted_entity.required_fields,
         subscription_type_id=subscription_type_id,
         updated_at=deleted_entity.updated_at,
         created_at=deleted_entity.created_at,
@@ -303,6 +343,13 @@ async def get_subscriptions_handler(
             if "label" in tag:
                 label = tag["label"]
 
+        normalized_entity_tags = [
+            f"{key}:{value}"
+            for tag in tags
+            for key, value in tag.items()
+            if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS
+        ]
+
         subscriptions.append(
             data.SubscriptionResourceData(
                 id=str(subscription.entity_id),
@@ -311,6 +358,8 @@ async def get_subscriptions_handler(
                 color=color,
                 label=label,
                 abi=subscription.secondary_fields.get("abi", None),
+                description=subscription.secondary_fields.get("description"),
+                tags=normalized_entity_tags,
                 subscription_type_id=subscription_type_id,
                 updated_at=subscription.updated_at,
                 created_at=subscription.created_at,
@@ -335,9 +384,6 @@ async def update_subscriptions_handler(
     request: Request,
     subscription_id: str,
     background_tasks: BackgroundTasks,
-    color: Optional[str] = Form(None),
-    label: Optional[str] = Form(None),
-    abi: Optional[str] = Form(None),
 ) -> data.SubscriptionResourceData:
     """
     Get user's subscriptions.
@@ -346,9 +392,17 @@ async def update_subscriptions_handler(
 
     user = request.state.user
 
-    update_required_fields = []
+    form = await request.form()
+    try:
+        form_data = data.UpdateSubscriptionRequest(**form)
+    except Exception as e:
+        raise MoonstreamHTTPException(status_code=400, detail=str(e))
 
-    update_secondary_fields = {}
+    color = form_data.color
+    label = form_data.label
+    abi = form_data.abi
+    description = form_data.description
+    tags = form_data.tags
 
     try:
         collection_id = get_entity_subscription_collection_id(
@@ -366,7 +420,13 @@ async def update_subscriptions_handler(
 
         subscription_type_id = None
 
-        update_required_fields = subscription_entity.required_fields
+        update_required_fields = [
+            field
+            for field in subscription_entity.required_fields
+            if any(key in field for key in MOONSTREAM_ENTITIES_RESERVED_TAGS)
+        ]
+
+        update_secondary_fields = subscription_entity.secondary_fields
 
         for field in update_required_fields:
             if "subscription_type_id" in field:
@@ -377,7 +437,7 @@ async def update_subscriptions_handler(
                 f"Subscription entity {subscription_id} in collection {collection_id} has no subscription_type_id malformed subscription entity"
             )
             raise MoonstreamHTTPException(
-                status_code=404,
+                status_code=409,
                 detail="Not valid subscription entity",
             )
 
@@ -394,13 +454,19 @@ async def update_subscriptions_handler(
         raise MoonstreamHTTPException(status_code=500, internal_error=e)
 
     for field in update_required_fields:
-        if "color" in field and color is not None:
+        if "color" in field:
-            field["color"] = color
+            if color is not None:
+                field["color"] = color
+            else:
+                color = field["color"]
 
-        if "label" in field and label is not None:
+        if "label" in field:
-            field["label"] = label
+            if label is not None:
+                field["label"] = label
+            else:
+                label = field["label"]
 
-    if abi:
+    if abi is not None:
         try:
             json_abi = json.loads(abi)
         except json.JSONDecodeError:
@@ -414,9 +480,19 @@ async def update_subscriptions_handler(
         hash = hashlib.md5(abi_string.encode("utf-8")).hexdigest()
 
         update_secondary_fields["abi_hash"] = hash
-    else:
-        update_secondary_fields = subscription_entity.secondary_fields
 
+    if description is not None:
+        update_secondary_fields["description"] = description
+
+    if tags:
+        allowed_required_fields = [
+            item
+            for item in tags
+            if not any(key in item for key in MOONSTREAM_ENTITIES_RESERVED_TAGS)
+        ]
+
+        if allowed_required_fields:
+            update_required_fields.extend(allowed_required_fields)
     try:
         subscription = ec.update_entity(
             token=token,
@@ -441,6 +517,13 @@ async def update_subscriptions_handler(
             subscription.address,
         )
 
+    normalized_entity_tags = [
+        f"{key}:{value}"
+        for tag in subscription.required_fields
+        for key, value in tag.items()
+        if key not in MOONSTREAM_ENTITIES_RESERVED_TAGS
+    ]
+
     return data.SubscriptionResourceData(
         id=str(subscription.entity_id),
         user_id=str(user.id),
@@ -448,6 +531,8 @@ async def update_subscriptions_handler(
         color=color,
         label=label,
         abi=subscription.secondary_fields.get("abi"),
+        description=subscription.secondary_fields.get("description"),
+        tags=normalized_entity_tags,
         subscription_type_id=subscription_type_id,
         updated_at=subscription_entity.updated_at,
         created_at=subscription_entity.created_at,

moonstreamapi/moonstreamapi/settings.py

@@ -137,6 +137,13 @@ MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get(
 if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "":
     raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set")
 
+MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI = os.environ.get(
+    "MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI", ""
+)
+if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "":
+    raise Exception(
+        "MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set"
+    )
 
 MOONSTREAM_S3_QUERIES_BUCKET = os.environ.get("MOONSTREAM_S3_QUERIES_BUCKET", "")
 if MOONSTREAM_S3_QUERIES_BUCKET == "":
@@ -150,6 +157,16 @@ if MOONSTREAM_S3_QUERIES_BUCKET_PREFIX == "":
         "MOONSTREAM_S3_QUERIES_BUCKET_PREFIX environment variable must be set"
     )
 
+# Entities reserved tags
+MOONSTREAM_ENTITIES_RESERVED_TAGS = [
+    "type",
+    "subscription_type_id",
+    "color",
+    "label",
+    "user_id",
+    "address",
+    "blockchain",
+]
 
 ## Moonstream resources types
 

moonstreamapi/moonstreamapi/version.py

@@ -2,4 +2,4 @@
 Moonstream library and API version.
 """
 
-MOONSTREAMAPI_VERSION = "0.2.6"
+MOONSTREAMAPI_VERSION = "0.2.7"

moonstreamapi/moonstreamapi/web3_provider.py

@@ -19,6 +19,7 @@ from .settings import (
     MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
     MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
     MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
+    MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI,
     multicall_contracts,
     multicall_contract_abi,
 )
@@ -69,6 +70,8 @@ def connect(
             web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI
         elif blockchain_type == AvailableBlockchainType.WYRM:
             web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
+        elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+            web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
         else:
             raise Exception("Wrong blockchain type provided for web3 URI")
 

moonstreamapi/requirements.txt

@@ -37,7 +37,7 @@ lru-dict==1.1.8
 Mako==1.2.3
 MarkupSafe==2.1.1
 moonstream-entity==0.0.5
-moonstreamdb==0.3.3
+moonstreamdb==0.3.4
 multiaddr==0.0.9
 multidict==6.0.2
 netaddr==0.8.0

moonstreamapi/setup.py

@@ -16,7 +16,7 @@ setup(
         "bugout>=0.2.9",
         "moonstream-entity>=0.0.5",
         "fastapi",
-        "moonstreamdb>=0.3.3",
+        "moonstreamdb>=0.3.4",
         "humbug",
         "pydantic",
         "pyevmasm",

moonstreamdb/.isort.cfg

@@ -0,0 +1,3 @@
+[settings]
+profile = black
+multi_line_output = 3

moonstreamdb/alembic/env.py

@@ -1,7 +1,6 @@
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config
+from sqlalchemy import engine_from_config, pool
-from sqlalchemy import pool
 
 from alembic import context
 
@@ -26,18 +25,27 @@ target_metadata = MoonstreamBase.metadata
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 from moonstreamdb.models import (
-    EthereumBlock,
-    EthereumTransaction,
-    EthereumLabel,
-    PolygonBlock,
-    PolygonTransaction,
-    PolygonLabel,
-    MumbaiBlock,
-    MumbaiTransaction,
-    MumbaiLabel,
-    ESDFunctionSignature,
     ESDEventSignature,
+    ESDFunctionSignature,
+    EthereumBlock,
+    EthereumLabel,
+    EthereumTransaction,
+    MumbaiBlock,
+    MumbaiLabel,
+    MumbaiTransaction,
     OpenSeaCrawlingState,
+    PolygonBlock,
+    PolygonLabel,
+    PolygonTransaction,
+    WyrmBlock,
+    WyrmLabel,
+    WyrmTransaction,
+    XDaiBlock,
+    XDaiLabel,
+    XDaiTransaction,
+    ZkSyncEraTestnetBlock,
+    ZkSyncEraTestnetLabel,
+    ZkSyncEraTestnetTransaction,
 )
 
 
@@ -55,6 +63,15 @@ def include_symbol(tablename, schema):
         ESDFunctionSignature.__tablename__,
         ESDEventSignature.__tablename__,
         OpenSeaCrawlingState.__tablename__,
+        WyrmBlock.__tablename__,
+        WyrmLabel.__tablename__,
+        WyrmTransaction.__tablename__,
+        XDaiBlock.__tablename__,
+        XDaiLabel.__tablename__,
+        XDaiTransaction.__tablename__,
+        ZkSyncEraTestnetBlock.__tablename__,
+        ZkSyncEraTestnetLabel.__tablename__,
+        ZkSyncEraTestnetTransaction.__tablename__,
     }
 
 

moonstreamdb/alembic/versions/e4a796c0407d_zksync_era_model.py

@@ -0,0 +1,122 @@
+"""ZkSync Era model
+
+Revision ID: e4a796c0407d
+Revises: c413d5265f76
+Create Date: 2023-07-12 12:27:12.814892
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'e4a796c0407d'
+down_revision = 'c413d5265f76'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('zksync_era_testnet_blocks',
+    sa.Column('block_number', sa.BigInteger(), nullable=False),
+    sa.Column('difficulty', sa.BigInteger(), nullable=True),
+    sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True),
+    sa.Column('gas_limit', sa.BigInteger(), nullable=True),
+    sa.Column('gas_used', sa.BigInteger(), nullable=True),
+    sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
+    sa.Column('hash', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True),
+    sa.Column('miner', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('uncles', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('size', sa.Integer(), nullable=True),
+    sa.Column('state_root', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('timestamp', sa.BigInteger(), nullable=True),
+    sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
+    sa.Column('mix_hash', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('sha3_uncles', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('l1_batch_number', sa.BigInteger(), nullable=True),
+    sa.Column('l1_batch_timestamp', sa.BigInteger(), nullable=True),
+    sa.PrimaryKeyConstraint('block_number', name=op.f('pk_zksync_era_testnet_blocks'))
+    )
+    op.create_index(op.f('ix_zksync_era_testnet_blocks_block_number'), 'zksync_era_testnet_blocks', ['block_number'], unique=True)
+    op.create_index(op.f('ix_zksync_era_testnet_blocks_hash'), 'zksync_era_testnet_blocks', ['hash'], unique=False)
+    op.create_index(op.f('ix_zksync_era_testnet_blocks_timestamp'), 'zksync_era_testnet_blocks', ['timestamp'], unique=False)
+    op.create_table('zksync_era_testnet_labels',
+    sa.Column('id', sa.UUID(), nullable=False),
+    sa.Column('label', sa.VARCHAR(length=256), nullable=False),
+    sa.Column('block_number', sa.BigInteger(), nullable=True),
+    sa.Column('address', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('block_timestamp', sa.BigInteger(), nullable=True),
+    sa.Column('log_index', sa.Integer(), nullable=True),
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_zksync_era_testnet_labels')),
+    sa.UniqueConstraint('id', name=op.f('uq_zksync_era_testnet_labels_id'))
+    )
+    op.create_index(op.f('ix_zksync_era_testnet_labels_address'), 'zksync_era_testnet_labels', ['address'], unique=False)
+    op.create_index('ix_zksync_era_testnet_labels_address_block_number', 'zksync_era_testnet_labels', ['address', 'block_number'], unique=False)
+    op.create_index('ix_zksync_era_testnet_labels_address_block_timestamp', 'zksync_era_testnet_labels', ['address', 'block_timestamp'], unique=False)
+    op.create_index(op.f('ix_zksync_era_testnet_labels_block_number'), 'zksync_era_testnet_labels', ['block_number'], unique=False)
+    op.create_index(op.f('ix_zksync_era_testnet_labels_block_timestamp'), 'zksync_era_testnet_labels', ['block_timestamp'], unique=False)
+    op.create_index(op.f('ix_zksync_era_testnet_labels_label'), 'zksync_era_testnet_labels', ['label'], unique=False)
+    op.create_index(op.f('ix_zksync_era_testnet_labels_transaction_hash'), 'zksync_era_testnet_labels', ['transaction_hash'], unique=False)
+    op.create_table('zksync_era_testnet_transactions',
+    sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
+    sa.Column('block_number', sa.BigInteger(), nullable=False),
+    sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
+    sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
+    sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
+    sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
+    sa.Column('input', sa.Text(), nullable=True),
+    sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
+    sa.Column('transaction_index', sa.BigInteger(), nullable=True),
+    sa.Column('transaction_type', sa.Integer(), nullable=True),
+    sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
+    sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
+    sa.Column('l1_batch_number', sa.BigInteger(), nullable=True),
+    sa.Column('l1_batch_tx_index', sa.BigInteger(), nullable=True),
+    sa.ForeignKeyConstraint(['block_number'], ['zksync_era_testnet_blocks.block_number'], name=op.f('fk_zksync_era_testnet_transactions_block_number_zksync_era_testnet_blocks'), ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('hash', name=op.f('pk_zksync_era_testnet_transactions'))
+    )
+    op.create_index(op.f('ix_zksync_era_testnet_transactions_block_number'), 'zksync_era_testnet_transactions', ['block_number'], unique=False)
+    op.create_index(op.f('ix_zksync_era_testnet_transactions_from_address'), 'zksync_era_testnet_transactions', ['from_address'], unique=False)
+    op.create_index(op.f('ix_zksync_era_testnet_transactions_gas'), 'zksync_era_testnet_transactions', ['gas'], unique=False)
+    op.create_index(op.f('ix_zksync_era_testnet_transactions_gas_price'), 'zksync_era_testnet_transactions', ['gas_price'], unique=False)
+    op.create_index(op.f('ix_zksync_era_testnet_transactions_hash'), 'zksync_era_testnet_transactions', ['hash'], unique=True)
+    op.create_index(op.f('ix_zksync_era_testnet_transactions_to_address'), 'zksync_era_testnet_transactions', ['to_address'], unique=False)
+    op.create_index(op.f('ix_zksync_era_testnet_transactions_value'), 'zksync_era_testnet_transactions', ['value'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_zksync_era_testnet_transactions_value'), table_name='zksync_era_testnet_transactions')
+    op.drop_index(op.f('ix_zksync_era_testnet_transactions_to_address'), table_name='zksync_era_testnet_transactions')
+    op.drop_index(op.f('ix_zksync_era_testnet_transactions_hash'), table_name='zksync_era_testnet_transactions')
+    op.drop_index(op.f('ix_zksync_era_testnet_transactions_gas_price'), table_name='zksync_era_testnet_transactions')
+    op.drop_index(op.f('ix_zksync_era_testnet_transactions_gas'), table_name='zksync_era_testnet_transactions')
+    op.drop_index(op.f('ix_zksync_era_testnet_transactions_from_address'), table_name='zksync_era_testnet_transactions')
+    op.drop_index(op.f('ix_zksync_era_testnet_transactions_block_number'), table_name='zksync_era_testnet_transactions')
+    op.drop_table('zksync_era_testnet_transactions')
+    op.drop_index(op.f('ix_zksync_era_testnet_labels_transaction_hash'), table_name='zksync_era_testnet_labels')
+    op.drop_index(op.f('ix_zksync_era_testnet_labels_label'), table_name='zksync_era_testnet_labels')
+    op.drop_index(op.f('ix_zksync_era_testnet_labels_block_timestamp'), table_name='zksync_era_testnet_labels')
+    op.drop_index(op.f('ix_zksync_era_testnet_labels_block_number'), table_name='zksync_era_testnet_labels')
+    op.drop_index('ix_zksync_era_testnet_labels_address_block_timestamp', table_name='zksync_era_testnet_labels')
+    op.drop_index('ix_zksync_era_testnet_labels_address_block_number', table_name='zksync_era_testnet_labels')
+    op.drop_index(op.f('ix_zksync_era_testnet_labels_address'), table_name='zksync_era_testnet_labels')
+    op.drop_table('zksync_era_testnet_labels')
+    op.drop_index(op.f('ix_zksync_era_testnet_blocks_timestamp'), table_name='zksync_era_testnet_blocks')
+    op.drop_index(op.f('ix_zksync_era_testnet_blocks_hash'), table_name='zksync_era_testnet_blocks')
+    op.drop_index(op.f('ix_zksync_era_testnet_blocks_block_number'), table_name='zksync_era_testnet_blocks')
+    op.drop_table('zksync_era_testnet_blocks')
+    # ### end Alembic commands ###

moonstreamdb/moonstreamdb/blockchain.py

@@ -1,26 +1,27 @@
-from .db import yield_db_session, yield_db_session_ctx
+from enum import Enum
+from typing import Type, Union
+
 from .models import (
     EthereumBlock,
     EthereumLabel,
     EthereumTransaction,
-    PolygonBlock,
-    PolygonLabel,
-    PolygonTransaction,
     MumbaiBlock,
     MumbaiLabel,
     MumbaiTransaction,
-    XDaiBlock,
+    PolygonBlock,
-    XDaiLabel,
+    PolygonLabel,
-    XDaiTransaction,
+    PolygonTransaction,
     WyrmBlock,
     WyrmLabel,
     WyrmTransaction,
+    XDaiBlock,
+    XDaiLabel,
+    XDaiTransaction,
+    ZkSyncEraTestnetBlock,
+    ZkSyncEraTestnetLabel,
+    ZkSyncEraTestnetTransaction,
 )
 
-from enum import Enum
-
-from typing import Type, Union
-
 
 class AvailableBlockchainType(Enum):
     ETHEREUM = "ethereum"
@@ -28,17 +29,34 @@ class AvailableBlockchainType(Enum):
     MUMBAI = "mumbai"
     XDAI = "xdai"
     WYRM = "wyrm"
+    ZKSYNC_ERA_TESTNET = "zksync_era_testnet"
 
 
 def get_block_model(
     blockchain_type: AvailableBlockchainType,
-) -> Type[Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock, WyrmBlock]]:
+) -> Type[
+    Union[
+        EthereumBlock,
+        PolygonBlock,
+        MumbaiBlock,
+        XDaiBlock,
+        WyrmBlock,
+        ZkSyncEraTestnetBlock,
+    ]
+]:
     """
     Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm
     set proper blocks model.
     """
     block_model: Type[
-        Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock, WyrmBlock]
+        Union[
+            EthereumBlock,
+            PolygonBlock,
+            MumbaiBlock,
+            XDaiBlock,
+            WyrmBlock,
+            ZkSyncEraTestnetBlock,
+        ]
     ]
     if blockchain_type == AvailableBlockchainType.ETHEREUM:
         block_model = EthereumBlock
@@ -50,6 +68,8 @@ def get_block_model(
         block_model = XDaiBlock
     elif blockchain_type == AvailableBlockchainType.WYRM:
         block_model = WyrmBlock
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+        block_model = ZkSyncEraTestnetBlock
     else:
         raise Exception("Unsupported blockchain type provided")
 
@@ -58,13 +78,29 @@ def get_block_model(
 
 def get_label_model(
     blockchain_type: AvailableBlockchainType,
-) -> Type[Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel, WyrmLabel]]:
+) -> Type[
+    Union[
+        EthereumLabel,
+        PolygonLabel,
+        MumbaiLabel,
+        XDaiLabel,
+        WyrmLabel,
+        ZkSyncEraTestnetLabel,
+    ]
+]:
     """
     Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm
     set proper block label model.
     """
     label_model: Type[
-        Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel, WyrmLabel]
+        Union[
+            EthereumLabel,
+            PolygonLabel,
+            MumbaiLabel,
+            XDaiLabel,
+            WyrmLabel,
+            ZkSyncEraTestnetLabel,
+        ]
     ]
     if blockchain_type == AvailableBlockchainType.ETHEREUM:
         label_model = EthereumLabel
@@ -76,6 +112,8 @@ def get_label_model(
         label_model = XDaiLabel
     elif blockchain_type == AvailableBlockchainType.WYRM:
         label_model = WyrmLabel
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+        label_model = ZkSyncEraTestnetLabel
     else:
         raise Exception("Unsupported blockchain type provided")
 
@@ -91,6 +129,7 @@ def get_transaction_model(
         MumbaiTransaction,
         XDaiTransaction,
         WyrmTransaction,
+        ZkSyncEraTestnetTransaction,
     ]
 ]:
     """
@@ -104,6 +143,7 @@ def get_transaction_model(
             MumbaiTransaction,
             XDaiTransaction,
             WyrmTransaction,
+            ZkSyncEraTestnetTransaction,
         ]
     ]
     if blockchain_type == AvailableBlockchainType.ETHEREUM:
@@ -116,6 +156,8 @@ def get_transaction_model(
         transaction_model = XDaiTransaction
     elif blockchain_type == AvailableBlockchainType.WYRM:
         transaction_model = WyrmTransaction
+    elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
+        transaction_model = ZkSyncEraTestnetTransaction
     else:
         raise Exception("Unsupported blockchain type provided")
 

moonstreamdb/moonstreamdb/db.py

@@ -1,12 +1,12 @@
 """
 Moonstream database connection.
 """
-from contextlib import contextmanager
 import os
+from contextlib import contextmanager
 from typing import Generator
 
 from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker, Session
+from sqlalchemy.orm import Session, sessionmaker
 
 MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI")
 if MOONSTREAM_DB_URI is None:

moonstreamdb/moonstreamdb/models.py

@@ -1,21 +1,21 @@
 import uuid
 
-from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import (
+    VARCHAR,
     BigInteger,
     Column,
     DateTime,
+    ForeignKey,
     Index,
     Integer,
-    ForeignKey,
     MetaData,
     Numeric,
     Text,
-    VARCHAR,
 )
 from sqlalchemy.dialects.postgresql import JSONB, UUID
-from sqlalchemy.sql import expression
 from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.sql import expression
 
 """
 Naming conventions doc
@@ -614,6 +614,137 @@ class WyrmLabel(Base):  # type: ignore
     )
 
 
+class ZkSyncEraTestnetBlock(Base):  # type: ignore
+    __tablename__ = "zksync_era_testnet_blocks"
+
+    block_number = Column(
+        BigInteger, primary_key=True, unique=True, nullable=False, index=True
+    )
+    difficulty = Column(BigInteger)
+    extra_data = Column(VARCHAR(128))
+    gas_limit = Column(BigInteger)
+    gas_used = Column(BigInteger)
+    base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
+    hash = Column(VARCHAR(256), index=True)
+    logs_bloom = Column(VARCHAR(1024))
+    miner = Column(VARCHAR(256))
+    nonce = Column(VARCHAR(256))
+    parent_hash = Column(VARCHAR(256))
+    receipt_root = Column(VARCHAR(256))
+    uncles = Column(VARCHAR(256))
+    size = Column(Integer)
+    state_root = Column(VARCHAR(256))
+    timestamp = Column(BigInteger, index=True)
+    total_difficulty = Column(VARCHAR(256))
+    transactions_root = Column(VARCHAR(256))
+
+    indexed_at = Column(
+        DateTime(timezone=True), server_default=utcnow(), nullable=False
+    )
+
+    mix_hash = Column(VARCHAR(256), nullable=True)
+    sha3_uncles = Column(VARCHAR(256), nullable=True)
+
+    l1_batch_number = Column(BigInteger, nullable=True)
+    l1_batch_timestamp = Column(BigInteger, nullable=True)
+
+
+class ZkSyncEraTestnetTransaction(Base):  # type: ignore
+    __tablename__ = "zksync_era_testnet_transactions"
+
+    hash = Column(
+        VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
+    )
+    block_number = Column(
+        BigInteger,
+        ForeignKey("zksync_era_testnet_blocks.block_number", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    from_address = Column(VARCHAR(256), index=True)
+    to_address = Column(VARCHAR(256), index=True)
+    gas = Column(Numeric(precision=78, scale=0), index=True)
+    gas_price = Column(Numeric(precision=78, scale=0), index=True)
+    max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
+    max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
+    input = Column(Text)
+    nonce = Column(VARCHAR(256))
+    transaction_index = Column(BigInteger)
+    transaction_type = Column(Integer, nullable=True)
+    value = Column(Numeric(precision=78, scale=0), index=True)
+
+    indexed_at = Column(
+        DateTime(timezone=True), server_default=utcnow(), nullable=False
+    )
+
+    l1_batch_number = Column(BigInteger, nullable=True)
+    l1_batch_tx_index = Column(BigInteger, nullable=True)
+
+
+class ZkSyncEraTestnetLabel(Base):  # type: ignore
+    """
+    Example of label_data:
+        {
+            "label": "ERC20",
+            "label_data": {
+                "name": "Uniswap",
+                "symbol": "UNI"
+            }
+        },
+        {
+            "label": "Exchange"
+            "label_data": {...}
+        }
+    """
+
+    __tablename__ = "zksync_era_testnet_labels"
+
+    __table_args__ = (
+        Index(
+            "ix_zksync_era_testnet_labels_address_block_number",
+            "address",
+            "block_number",
+            unique=False,
+        ),
+        Index(
+            "ix_zksync_era_testnet_labels_address_block_timestamp",
+            "address",
+            "block_timestamp",
+            unique=False,
+        ),
+    )
+
+    id = Column(
+        UUID(as_uuid=True),
+        primary_key=True,
+        default=uuid.uuid4,
+        unique=True,
+        nullable=False,
+    )
+    label = Column(VARCHAR(256), nullable=False, index=True)
+    block_number = Column(
+        BigInteger,
+        nullable=True,
+        index=True,
+    )
+    address = Column(
+        VARCHAR(256),
+        nullable=True,
+        index=True,
+    )
+    transaction_hash = Column(
+        VARCHAR(256),
+        nullable=True,
+        index=True,
+    )
+    label_data = Column(JSONB, nullable=True)
+    block_timestamp = Column(BigInteger, index=True)
+    log_index = Column(Integer, nullable=True)
+    created_at = Column(
+        DateTime(timezone=True), server_default=utcnow(), nullable=False
+    )
+
+
 class ESDFunctionSignature(Base):  # type: ignore
     """
     Function signature from blockchain (Ethereum/Polygon) Signature Database.

moonstreamdb/moonstreamdb/networks.py

@@ -18,6 +18,9 @@ from .models import (
     XDaiBlock,
     XDaiLabel,
     XDaiTransaction,
+    ZkSyncEraTestnetBlock,
+    ZkSyncEraTestnetLabel,
+    ZkSyncEraTestnetTransaction,
 )
 
 
@@ -27,6 +30,7 @@ class Network(Enum):
     mumbai = "mumbai"
     xdai = "xdai"
     wyrm = "wyrm"
+    zksync_era_testnet = "zksync_era_testnet"
 
 
 tx_raw_types = Union[
@@ -35,6 +39,7 @@ tx_raw_types = Union[
     PolygonTransaction,
     WyrmTransaction,
     XDaiTransaction,
+    ZkSyncEraTestnetTransaction,
 ]
 
 MODELS: Dict[Network, Dict[str, Base]] = {
@@ -63,4 +68,9 @@ MODELS: Dict[Network, Dict[str, Base]] = {
         "labels": WyrmLabel,
         "transactions": WyrmTransaction,
     },
+    Network.zksync_era_testnet: {
+        "blocks": ZkSyncEraTestnetBlock,
+        "labels": ZkSyncEraTestnetLabel,
+        "transactions": ZkSyncEraTestnetTransaction,
+    },
 }

moonstreamdb/moonstreamdb/version.py

@@ -2,4 +2,4 @@
 Moonstream database version.
 """
 
-MOONSTREAMDB_VERSION = "0.3.3"
+MOONSTREAMDB_VERSION = "0.3.4"

nodebalancer/README.md

@@ -2,7 +2,7 @@
 
 ## Installation
 
--   Prepare environment variables, according with `sample.env`.
+-   Prepare environment variables, according to `sample.env`.
 -   Build application
 
 ```bash
@@ -60,14 +60,14 @@ This command will return a list of bugout resources of registered users to acces
 ]
 ```
 
-`access_id` - token which allow access to nodebalancer, could be specified in both ways:
+`access_id` - token which allows access to nodebalancer, could be specified in both ways:
 
 -   as a header `x-moonstream-access-id` with value `access_id`
 -   as query parameter `access_id=access_id`
 
-`blockchain_access` - boolean which allow you or not to have access to blockchain node, otherwise you will be redirected to database
+`blockchain_access` - boolean which allows you or not to have access to blockchain node, otherwise you will be redirected to database
 
-`extended_methods` - boolean which allow you to call not whitelisted method to blockchain node, by default for new user this is equal to `false`
+`extended_methods` - boolean which allows you to call not whitelisted method to blockchain node, by default for new user this is equal to `false`
 
 ### server
 

nodebalancer/cmd/nodebalancer/blockchain.go

@@ -36,6 +36,27 @@ var (
 		"net_version":   true,
 
 		"web3_clientVersion": true,
+
+		// zksync methods
+		"zks_estimateFee": true,
+		"zks_estimateGasL1ToL2": true,
+		"zks_getAllAccountBalances": true,
+		"zks_getBlockDetails": true,
+		"zks_getBridgeContracts": true,
+		"zks_getBytecodeByHash": true,
+		"zks_getConfirmedTokens": true,
+		"zks_getL1BatchBlockRange": true,
+		"zks_getL1BatchDetails": true,
+		"zks_getL2ToL1LogProof": true,
+		"zks_getL2ToL1MsgProof": true,
+		"zks_getMainContract": true,
+		"zks_getRawBlockTransactions": true,
+		"zks_getTestnetPaymaster": true,
+		"zks_getTokenPrice": true,
+		"zks_getTransactionDetails": true,
+		"zks_L1BatchNumber": true,
+		"zks_L1ChainId": true,
+
 	}
 )