Merge branch 'main' into biling-cli

crawlers/deploy/zksync-era-testnet-missing.service

@@ -1,11 +0,0 @@
-[Unit]
-Description=Fill missing blocks at ZkSync Era testnet database
-After=network.target
-
-[Service]
-Type=oneshot
-WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
-EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era_testnet -n
-CPUWeight=50
-SyslogIdentifier=zksync-era-testnet-missing

crawlers/deploy/zksync-era-testnet-missing.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=Fill missing blocks at ZkSync Era testnet database
-
-[Timer]
-OnBootSec=120s
-OnUnitActiveSec=15m
-
-[Install]
-WantedBy=timers.target

crawlers/deploy/zksync-era-testnet-moonworm-crawler.service

@@ -1,17 +0,0 @@
-[Unit]
-Description=ZkSync Era testnet moonworm crawler
-After=network.target
-StartLimitIntervalSec=300
-StartLimitBurst=3
-
-[Service]
-WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
-EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-Restart=on-failure
-RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era_testnet  --confirmations 20 --min-blocks-batch 20
-CPUWeight=70
-SyslogIdentifier=zksync-era-testnet-moonworm-crawler
-
-[Install]
-WantedBy=multi-user.target

crawlers/deploy/zksync-era-testnet-synchronize.service

@@ -1,17 +0,0 @@
-[Unit]
-Description=ZkSync Era testnet block with transactions synchronizer
-StartLimitIntervalSec=300
-StartLimitBurst=3
-After=network.target
-
-[Service]
-Restart=on-failure
-RestartSec=15s
-WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
-EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2
-CPUWeight=90
-SyslogIdentifier=zksync-era-testnet-synchronize
-
-[Install]
-WantedBy=multi-user.target

crawlers/mooncrawl/mooncrawl/blockchain.py

@@ -34,7 +34,6 @@ from .settings import (
     MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI,
     MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI,
     MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI,
-    MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI,
     WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
 )
 
@@ -67,8 +66,6 @@ def connect(
             web3_uri = MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.XDAI:
             web3_uri = MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI
-        elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
-            web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
             web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
@@ -151,6 +148,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
         block_obj.author = block.author
     if (
         blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET
+        or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA
         or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA
     ):
         block_obj.mix_hash = block.get("mixHash", "")
@@ -225,6 +223,7 @@ def add_block_transactions(
         )
         if (
             blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET
+            or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA
             or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA
         ):
             tx_obj.l1_batch_number = (

crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py

@@ -50,11 +50,14 @@ def _refetch_new_jobs(
     """
     Refetches new jobs from bugout journal, merges, and returns new jobs.
     """
-
+    max_created_at_event_job: Optional[int] = None
-    max_created_at_event_job = max(job.created_at for job in old_event_jobs)
+    max_created_at_function_call_job: Optional[int] = None
-    max_created_at_function_call_job = max(
+    if len(old_event_jobs) != 0:
-        job.created_at for job in old_function_call_jobs
+        max_created_at_event_job = max(job.created_at for job in old_event_jobs)
-    )
+    if len(old_function_call_jobs) != 0:
+        max_created_at_function_call_job = max(
+            job.created_at for job in old_function_call_jobs
+        )
 
     logger.info("Looking for new event crawl jobs.")
     old_event_jobs_length = len(old_event_jobs)

crawlers/mooncrawl/mooncrawl/settings.py

@@ -102,14 +102,6 @@ MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI = os.environ.get(
 if MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI == "":
     raise Exception("MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI env variable is not set")
 
-MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI = os.environ.get(
-    "MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI", ""
-)
-if MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI == "":
-    raise Exception(
-        "MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI env variable is not set"
-    )
-
 MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI = os.environ.get(
     "MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI", ""
 )
@@ -258,14 +250,6 @@ NB_DATA_SOURCE_HEADER = os.environ.get(
     "NB_DATA_SOURCE_HEADER", "x-node-balancer-data-source"
 )
 
-NB_CONTROLLER_ACCESS_ID: Optional[UUID] = None
-NB_CONTROLLER_ACCESS_ID_RAW = os.environ.get("NB_CONTROLLER_ACCESS_ID", "")
-try:
-    NB_CONTROLLER_ACCESS_ID = UUID(NB_CONTROLLER_ACCESS_ID_RAW)
-except:
-    pass
-
-
 # HTTPProvider for web3 client
 
 WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS = 600

crawlers/mooncrawl/mooncrawl/version.py

@@ -2,4 +2,4 @@
 Moonstream crawlers version.
 """
 
-MOONCRAWL_VERSION = "0.4.1"
+MOONCRAWL_VERSION = "0.4.2"

crawlers/mooncrawl/sample.env

@@ -25,7 +25,6 @@ export MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI="https://<connection_path_uri_to_
 export MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
@@ -34,7 +33,6 @@ export MOONSTREAM_NODE_XAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>
 export MOONSTREAM_NODE_XAI_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
 
 # AWS environment variables
 export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS_S3_bucket_for_smart_contracts>"

crawlers/mooncrawl/setup.py

@@ -37,7 +37,7 @@ setup(
         "bugout>=0.2.13",
         "chardet",
         "fastapi",
-        "moonstreamdb>=0.3.10",
+        "moonstreamdb>=0.3.11",
         "moonstream>=0.1.1",
         "moonworm[moonstream]>=0.6.2",
         "humbug",

moonstreamdb/alembic/versions/295ce155c811_fix_avalanche.py

@@ -0,0 +1,34 @@
+"""Fix avalanche
+
+Revision ID: 295ce155c811
+Revises: 26000a6bff7d
+Create Date: 2024-04-01 09:20:46.381074
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '295ce155c811'
+down_revision = '26000a6bff7d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('avalanche_blocks', 'block_extra_data')
+    op.drop_column('avalanche_fuji_blocks', 'block_extra_data')
+    op.add_column('avalanche_blocks', sa.Column('block_extra_data', sa.Text(), nullable=True))
+    op.add_column('avalanche_fuji_blocks', sa.Column('block_extra_data', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('avalanche_blocks', 'block_extra_data')
+    op.drop_column('avalanche_fuji_blocks', 'block_extra_data')
+    op.add_column('avalanche_blocks', sa.Column('block_extra_data', sa.VARCHAR(length=256), nullable=True))
+    op.add_column('avalanche_fuji_blocks', sa.Column('block_extra_data', sa.VARCHAR(length=256), nullable=True))
+    # ### end Alembic commands ###

moonstreamdb/moonstreamdb/models.py

@@ -1406,7 +1406,7 @@ class AvalancheBlock(Base):  # type: ignore
     )
 
     mix_hash = Column(VARCHAR(256), nullable=True)
-    block_extra_data = Column(VARCHAR(256), nullable=True)
+    block_extra_data = Column(Text, nullable=True)
     block_gas_cost = Column(VARCHAR(256), nullable=True)
     ext_data_gas_used = Column(VARCHAR(256), nullable=True)
     ext_data_hash = Column(VARCHAR(256), nullable=True)

moonstreamdb/moonstreamdb/version.py

@@ -2,4 +2,4 @@
 Moonstream database version.
 """
 
-MOONSTREAMDB_VERSION = "0.3.10"
+MOONSTREAMDB_VERSION = "0.3.11"