mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #1043 from moonstream-to/fix-avalanche-db 

Fix avalanche
pull/831/head moonstreamdb/v0.3.11
Sergei Sumarokov 2024-04-01 12:56:14 +03:00 zatwierdzone przez GitHub
rodzic cbbac43d05 5ab43d398e
commit 19a17f83e2
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: B5690EEEBB952194
13 zmienionych plików z 48 dodań i 84 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

11
crawlers/deploy/zksync-era-testnet-missing.service
Wyświetl plik

@ -1,11 +0,0 @@
[Unit]
Description=Fill missing blocks at ZkSync Era testnet database
After=network.target
[Service]
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era_testnet -n
CPUWeight=50
SyslogIdentifier=zksync-era-testnet-missing

9
crawlers/deploy/zksync-era-testnet-missing.timer
Wyświetl plik

@ -1,9 +0,0 @@
[Unit]
Description=Fill missing blocks at ZkSync Era testnet database
[Timer]
OnBootSec=120s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

17
crawlers/deploy/zksync-era-testnet-moonworm-crawler.service
Wyświetl plik

@ -1,17 +0,0 @@
[Unit]
Description=ZkSync Era testnet moonworm crawler
After=network.target
StartLimitIntervalSec=300
StartLimitBurst=3
[Service]
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era_testnet --confirmations 20 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=zksync-era-testnet-moonworm-crawler
[Install]
WantedBy=multi-user.target

17
crawlers/deploy/zksync-era-testnet-synchronize.service
Wyświetl plik

@ -1,17 +0,0 @@
[Unit]
Description=ZkSync Era testnet block with transactions synchronizer
StartLimitIntervalSec=300
StartLimitBurst=3
After=network.target
[Service]
Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2
CPUWeight=90
SyslogIdentifier=zksync-era-testnet-synchronize
[Install]
WantedBy=multi-user.target

5
crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -34,7 +34,6 @@ from .settings import (
MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI,
MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI,
MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI,
MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI,
WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
)
@ -67,8 +66,6 @@ def connect(
web3_uri = MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.XDAI:
web3_uri = MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
@ -151,6 +148,7 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
block_obj.author = block.author
if (
blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET
or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA
or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA
):
block_obj.mix_hash = block.get("mixHash", "")
@ -225,6 +223,7 @@ def add_block_transactions(
)
if (
blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET
or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA
or blockchain_type == AvailableBlockchainType.ZKSYNC_ERA
):
tx_obj.l1_batch_number = (

13
crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py
Wyświetl plik

@ -50,11 +50,14 @@ def _refetch_new_jobs(
"""
Refetches new jobs from bugout journal, merges, and returns new jobs.
"""
max_created_at_event_job = max(job.created_at for job in old_event_jobs)
max_created_at_function_call_job = max(
job.created_at for job in old_function_call_jobs
)
max_created_at_event_job: Optional[int] = None
max_created_at_function_call_job: Optional[int] = None
if len(old_event_jobs) != 0:
max_created_at_event_job = max(job.created_at for job in old_event_jobs)
if len(old_function_call_jobs) != 0:
max_created_at_function_call_job = max(
job.created_at for job in old_function_call_jobs
)
logger.info("Looking for new event crawl jobs.")
old_event_jobs_length = len(old_event_jobs)

16
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -102,14 +102,6 @@ MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI = os.environ.get(
if MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI", ""
)
if MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI == "":
raise Exception(
"MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI env variable is not set"
)
MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI", ""
)
@ -258,14 +250,6 @@ NB_DATA_SOURCE_HEADER = os.environ.get(
"NB_DATA_SOURCE_HEADER", "x-node-balancer-data-source"
)
NB_CONTROLLER_ACCESS_ID: Optional[UUID] = None
NB_CONTROLLER_ACCESS_ID_RAW = os.environ.get("NB_CONTROLLER_ACCESS_ID", "")
try:
NB_CONTROLLER_ACCESS_ID = UUID(NB_CONTROLLER_ACCESS_ID_RAW)
except:
pass
# HTTPProvider for web3 client
WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS = 600

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.4.1"
MOONCRAWL_VERSION = "0.4.2"

2
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -25,7 +25,6 @@ export MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI="https://<connection_path_uri_to_
export MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
@ -34,7 +33,6 @@ export MOONSTREAM_NODE_XAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>
export MOONSTREAM_NODE_XAI_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
# AWS environment variables
export MOONSTREAM_S3_SMARTCONTRACTS_BUCKET="<AWS_S3_bucket_for_smart_contracts>"

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -37,7 +37,7 @@ setup(
"bugout>=0.2.13",
"chardet",
"fastapi",
"moonstreamdb>=0.3.10",
"moonstreamdb>=0.3.11",
"moonstream>=0.1.1",
"moonworm[moonstream]>=0.6.2",
"humbug",

34
moonstreamdb/alembic/versions/295ce155c811_fix_avalanche.py 100644
Wyświetl plik

@ -0,0 +1,34 @@
"""Fix avalanche
Revision ID: 295ce155c811
Revises: 26000a6bff7d
Create Date: 2024-04-01 09:20:46.381074
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '295ce155c811'
down_revision = '26000a6bff7d'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('avalanche_blocks', 'block_extra_data')
op.drop_column('avalanche_fuji_blocks', 'block_extra_data')
op.add_column('avalanche_blocks', sa.Column('block_extra_data', sa.Text(), nullable=True))
op.add_column('avalanche_fuji_blocks', sa.Column('block_extra_data', sa.Text(), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('avalanche_blocks', 'block_extra_data')
op.drop_column('avalanche_fuji_blocks', 'block_extra_data')
op.add_column('avalanche_blocks', sa.Column('block_extra_data', sa.VARCHAR(length=256), nullable=True))
op.add_column('avalanche_fuji_blocks', sa.Column('block_extra_data', sa.VARCHAR(length=256), nullable=True))
# ### end Alembic commands ###

2
moonstreamdb/moonstreamdb/models.py
Wyświetl plik

@ -1406,7 +1406,7 @@ class AvalancheBlock(Base): # type: ignore
)
mix_hash = Column(VARCHAR(256), nullable=True)
block_extra_data = Column(VARCHAR(256), nullable=True)
block_extra_data = Column(Text, nullable=True)
block_gas_cost = Column(VARCHAR(256), nullable=True)
ext_data_gas_used = Column(VARCHAR(256), nullable=True)
ext_data_hash = Column(VARCHAR(256), nullable=True)

2
moonstreamdb/moonstreamdb/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream database version.
"""
MOONSTREAMDB_VERSION = "0.3.10"
MOONSTREAMDB_VERSION = "0.3.11"