From b8846f49e2e971830080516b5abcda470971c75d Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 1 Apr 2024 09:10:17 +0000 Subject: [PATCH 1/4] Removed zksync-era-testnet services --- .../deploy/zksync-era-testnet-missing.service | 11 ----------- .../deploy/zksync-era-testnet-missing.timer | 9 --------- .../zksync-era-testnet-moonworm-crawler.service | 17 ----------------- .../zksync-era-testnet-synchronize.service | 17 ----------------- crawlers/mooncrawl/mooncrawl/blockchain.py | 5 ++--- crawlers/mooncrawl/mooncrawl/settings.py | 16 ---------------- crawlers/mooncrawl/sample.env | 2 -- 7 files changed, 2 insertions(+), 75 deletions(-) delete mode 100644 crawlers/deploy/zksync-era-testnet-missing.service delete mode 100644 crawlers/deploy/zksync-era-testnet-missing.timer delete mode 100644 crawlers/deploy/zksync-era-testnet-moonworm-crawler.service delete mode 100644 crawlers/deploy/zksync-era-testnet-synchronize.service diff --git a/crawlers/deploy/zksync-era-testnet-missing.service b/crawlers/deploy/zksync-era-testnet-missing.service deleted file mode 100644 index 1b49ec56..00000000 --- a/crawlers/deploy/zksync-era-testnet-missing.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Fill missing blocks at ZkSync Era testnet database -After=network.target - -[Service] -Type=oneshot -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era_testnet -n -CPUWeight=50 -SyslogIdentifier=zksync-era-testnet-missing \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-testnet-missing.timer b/crawlers/deploy/zksync-era-testnet-missing.timer deleted file mode 100644 index 5837fb09..00000000 --- a/crawlers/deploy/zksync-era-testnet-missing.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Fill missing blocks at ZkSync Era testnet database - -[Timer] -OnBootSec=120s -OnUnitActiveSec=15m - -[Install] -WantedBy=timers.target \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-testnet-moonworm-crawler.service b/crawlers/deploy/zksync-era-testnet-moonworm-crawler.service deleted file mode 100644 index a825a2ee..00000000 --- a/crawlers/deploy/zksync-era-testnet-moonworm-crawler.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=ZkSync Era testnet moonworm crawler -After=network.target -StartLimitIntervalSec=300 -StartLimitBurst=3 - -[Service] -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -Restart=on-failure -RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era_testnet --confirmations 20 --min-blocks-batch 20 -CPUWeight=70 -SyslogIdentifier=zksync-era-testnet-moonworm-crawler - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-testnet-synchronize.service b/crawlers/deploy/zksync-era-testnet-synchronize.service deleted file mode 100644 index 348ba1e0..00000000 --- a/crawlers/deploy/zksync-era-testnet-synchronize.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=ZkSync Era testnet block with transactions synchronizer -StartLimitIntervalSec=300 -StartLimitBurst=3 -After=network.target - -[Service] -Restart=on-failure -RestartSec=15s -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2 -CPUWeight=90 -SyslogIdentifier=zksync-era-testnet-synchronize - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index 1ab29a00..e5e97564 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -34,7 +34,6 @@ from .settings import ( MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI, MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI, MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI, - MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI, WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS, ) @@ -67,8 +66,6 @@ def connect( web3_uri = MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.XDAI: web3_uri = MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI - elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: - web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: @@ -151,6 +148,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) block_obj.author = block.author if ( blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET + or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA ): block_obj.mix_hash = block.get("mixHash", "") @@ -225,6 +223,7 @@ def add_block_transactions( ) if ( blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET + or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA ): tx_obj.l1_batch_number = ( diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index e6659d5e..9c479fcf 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -102,14 +102,6 @@ MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI = os.environ.get( if MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI == "": raise Exception("MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI env variable is not set") -MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI = os.environ.get( - "MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI", "" -) -if MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI == "": - raise Exception( - "MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI env variable is not set" - ) - MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI = os.environ.get( "MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI", "" ) @@ -258,14 +250,6 @@ NB_DATA_SOURCE_HEADER = os.environ.get( "NB_DATA_SOURCE_HEADER", "x-node-balancer-data-source" ) -NB_CONTROLLER_ACCESS_ID: Optional[UUID] = None -NB_CONTROLLER_ACCESS_ID_RAW = os.environ.get("NB_CONTROLLER_ACCESS_ID", "") -try: - NB_CONTROLLER_ACCESS_ID = UUID(NB_CONTROLLER_ACCESS_ID_RAW) -except: - pass - - # HTTPProvider for web3 client WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS = 600 diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 095e99e0..9819b89f 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -25,7 +25,6 @@ export MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI="https://" export MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI="https://" -export NB_CONTROLLER_ACCESS_ID="" # AWS environment variables export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="" From 8af25265b30f2b69305065a10df1be0a9b095c8c Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 1 Apr 2024 09:51:55 +0000 Subject: [PATCH 2/4] Moonworm crawler fix jobs max --- .../moonworm_crawler/continuous_crawler.py | 13 ++++++++----- crawlers/mooncrawl/mooncrawl/version.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index 698f98ac..823e6dda 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -50,11 +50,14 @@ def _refetch_new_jobs( """ Refetches new jobs from bugout journal, merges, and returns new jobs. """ - - max_created_at_event_job = max(job.created_at for job in old_event_jobs) - max_created_at_function_call_job = max( - job.created_at for job in old_function_call_jobs - ) + max_created_at_event_job: Optional[int] = None + max_created_at_function_call_job: Optional[int] = None + if len(old_event_jobs) != 0: + max_created_at_event_job = max(job.created_at for job in old_event_jobs) + if len(old_function_call_jobs) != 0: + max_created_at_function_call_job = max( + job.created_at for job in old_function_call_jobs + ) logger.info("Looking for new event crawl jobs.") old_event_jobs_length = len(old_event_jobs) diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index de0438c6..11ddf645 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.4.1" +MOONCRAWL_VERSION = "0.4.2" From e406fc5b8a001885c936b35910e19de53d6971a0 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 1 Apr 2024 09:54:21 +0000 Subject: [PATCH 3/4] Avalanche db long data fix --- .../versions/295ce155c811_fix_avalanche.py | 34 +++++++++++++++++++ moonstreamdb/moonstreamdb/models.py | 2 +- moonstreamdb/moonstreamdb/version.py | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 moonstreamdb/alembic/versions/295ce155c811_fix_avalanche.py diff --git a/moonstreamdb/alembic/versions/295ce155c811_fix_avalanche.py b/moonstreamdb/alembic/versions/295ce155c811_fix_avalanche.py new file mode 100644 index 00000000..68fb7385 --- /dev/null +++ b/moonstreamdb/alembic/versions/295ce155c811_fix_avalanche.py @@ -0,0 +1,34 @@ +"""Fix avalanche + +Revision ID: 295ce155c811 +Revises: 26000a6bff7d +Create Date: 2024-04-01 09:20:46.381074 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '295ce155c811' +down_revision = '26000a6bff7d' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('avalanche_blocks', 'block_extra_data') + op.drop_column('avalanche_fuji_blocks', 'block_extra_data') + op.add_column('avalanche_blocks', sa.Column('block_extra_data', sa.Text(), nullable=True)) + op.add_column('avalanche_fuji_blocks', sa.Column('block_extra_data', sa.Text(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('avalanche_blocks', 'block_extra_data') + op.drop_column('avalanche_fuji_blocks', 'block_extra_data') + op.add_column('avalanche_blocks', sa.Column('block_extra_data', sa.VARCHAR(length=256), nullable=True)) + op.add_column('avalanche_fuji_blocks', sa.Column('block_extra_data', sa.VARCHAR(length=256), nullable=True)) + # ### end Alembic commands ### diff --git a/moonstreamdb/moonstreamdb/models.py b/moonstreamdb/moonstreamdb/models.py index fdf47804..4d31a325 100644 --- a/moonstreamdb/moonstreamdb/models.py +++ b/moonstreamdb/moonstreamdb/models.py @@ -1406,7 +1406,7 @@ class AvalancheBlock(Base): # type: ignore ) mix_hash = Column(VARCHAR(256), nullable=True) - block_extra_data = Column(VARCHAR(256), nullable=True) + block_extra_data = Column(Text, nullable=True) block_gas_cost = Column(VARCHAR(256), nullable=True) ext_data_gas_used = Column(VARCHAR(256), nullable=True) ext_data_hash = Column(VARCHAR(256), nullable=True) diff --git a/moonstreamdb/moonstreamdb/version.py b/moonstreamdb/moonstreamdb/version.py index a6e795a6..377aaf31 100644 --- a/moonstreamdb/moonstreamdb/version.py +++ b/moonstreamdb/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.3.10" +MOONSTREAMDB_VERSION = "0.3.11" From 5ab43d398ef982189e1c2762b6943650cd83a1cd Mon Sep 17 00:00:00 2001 From: kompotkot Date: Mon, 1 Apr 2024 09:55:05 +0000 Subject: [PATCH 4/4] Bumped for dep moonstreamdb version --- crawlers/mooncrawl/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 8274173a..ce99baa9 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -37,7 +37,7 @@ setup( "bugout>=0.2.13", "chardet", "fastapi", - "moonstreamdb>=0.3.10", + "moonstreamdb>=0.3.11", "moonstream>=0.1.1", "moonworm[moonstream]>=0.6.2", "humbug",