diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 6707bff8..b6ac671f 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -106,6 +106,11 @@ WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transaction WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="wyrm-historical-crawl-events.service" WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="wyrm-historical-crawl-events.timer" +# ZkSync Era testnet +ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service" +ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE="zksync-era-testnet-missing.service" +ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE="zksync-era-testnet-missing.service" + set -eu echo @@ -499,3 +504,21 @@ cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/home/ubuntu/.c cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" + +# ZkSync Era +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet block with transactions syncronizer service definition with ${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet missing service and timer with: ${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}" \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-testnet-missing.service b/crawlers/deploy/zksync-era-testnet-missing.service new file mode 100644 index 00000000..0a1a26da --- /dev/null +++ b/crawlers/deploy/zksync-era-testnet-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at ZkSync Era testnet database +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era_testnet -n +CPUWeight=50 +SyslogIdentifier=zksync-era-testnet-missing \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-testnet-missing.timer b/crawlers/deploy/zksync-era-testnet-missing.timer new file mode 100644 index 00000000..5837fb09 --- /dev/null +++ b/crawlers/deploy/zksync-era-testnet-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at ZkSync Era testnet database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-testnet-synchronize.service b/crawlers/deploy/zksync-era-testnet-synchronize.service new file mode 100644 index 00000000..ad9822d3 --- /dev/null +++ b/crawlers/deploy/zksync-era-testnet-synchronize.service @@ -0,0 +1,17 @@ +[Unit] +Description=ZkSync Era testnet block with transactions synchronizer +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2 +CPUWeight=90 +SyslogIdentifier=zksync-era-testnet-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index e6656fd2..983c4f23 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -27,6 +27,7 @@ from .settings import ( MOONSTREAM_POLYGON_WEB3_PROVIDER_URI, MOONSTREAM_WYRM_WEB3_PROVIDER_URI, MOONSTREAM_XDAI_WEB3_PROVIDER_URI, + MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI, NB_ACCESS_ID_HEADER, NB_DATA_SOURCE_HEADER, WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS, @@ -70,6 +71,8 @@ def connect( web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI elif blockchain_type == AvailableBlockchainType.WYRM: web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: + web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI else: raise Exception("Wrong blockchain type provided for web3 URI") @@ -123,6 +126,11 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) ) if blockchain_type == AvailableBlockchainType.XDAI: block_obj.author = block.author + if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: + block_obj.mix_hash = block.get("mixHash", "") + block_obj.sha3_uncles = block.get("sha3Uncles", "") + block_obj.l1_batch_number = block.get("l1BatchNumber", None) + block_obj.l1_batch_timestamp = block.get("l1BatchTimestamp", None) db_session.add(block_obj) @@ -152,6 +160,9 @@ def add_block_transactions( transaction_type=int(tx["type"], 0) if tx.get("type") is not None else None, value=tx.value, ) + if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: + tx_obj.l1_batch_number = tx.get("l1BatchNumber", None) + tx_obj.l1_batch_tx_index = tx.get("l1BatchTxIndex", None) db_session.add(tx_obj) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index fa8249b6..3ab46897 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -130,6 +130,8 @@ def continuous_crawler( network = Network.xdai elif blockchain_type == AvailableBlockchainType.WYRM: network = Network.wyrm + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: + network = Network.zksync_era_testnet else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index de52d52e..0c7dfd54 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -35,6 +35,7 @@ class SubscriptionTypes(Enum): MUMBAI_BLOCKCHAIN = "mumbai_smartcontract" XDAI_BLOCKCHAIN = "xdai_smartcontract" WYRM_BLOCKCHAIN = "wyrm_smartcontract" + ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract" def abi_input_signature(input_abi: Dict[str, Any]) -> str: @@ -139,6 +140,8 @@ def blockchain_type_to_subscription_type( return SubscriptionTypes.XDAI_BLOCKCHAIN elif blockchain_type == AvailableBlockchainType.WYRM: return SubscriptionTypes.WYRM_BLOCKCHAIN + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: + return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py index 98e93b17..7bc27bd7 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py @@ -68,6 +68,8 @@ def function_call_crawler( network = Network.xdai elif blockchain_type == AvailableBlockchainType.WYRM: network = Network.wyrm + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: + network = Network.zksync_era_testnet else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index d9a24f70..b0d36b89 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -114,6 +114,14 @@ MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get( if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "": raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set") +MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI = os.environ.get( + "MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI", "" +) +if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "": + raise Exception( + "MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set" + ) + MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") try: diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 71cc3320..2a223bdd 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -48,6 +48,7 @@ subscription_id_by_blockchain = { "mumbai": "mumbai_smartcontract", "xdai": "xdai_smartcontract", "wyrm": "wyrm_smartcontract", + "zksync_era_testnet": "zksync_era_testnet_smartcontract", } blockchain_by_subscription_id = { @@ -56,11 +57,13 @@ blockchain_by_subscription_id = { "mumbai_blockchain": "mumbai", "xdai_blockchain": "xdai", "wyrm_blockchain": "wyrm", + "zksync_era_testnet_blockchain": "zksync_era_testnet", "ethereum_smartcontract": "ethereum", "polygon_smartcontract": "polygon", "mumbai_smartcontract": "mumbai", "xdai_smartcontract": "xdai", "wyrm_smartcontract": "wyrm", + "zksync_era_testnet_smartcontract": "zksync_era_testnet", } diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index 581a985f..f04f8d7c 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.3.1" +MOONCRAWL_VERSION = "0.3.2" diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index f1b863a0..57e91f94 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -25,6 +25,7 @@ export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://=0.2.8", "chardet", "fastapi", - "moonstreamdb>=0.3.3", + "moonstreamdb>=0.3.4", "moonstream>=0.1.1", "moonstream-entity>=0.0.5", "moonworm[moonstream]>=0.6.2", diff --git a/moonstreamdb/.isort.cfg b/moonstreamdb/.isort.cfg new file mode 100644 index 00000000..81d54de1 --- /dev/null +++ b/moonstreamdb/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +profile = black +multi_line_output = 3 \ No newline at end of file diff --git a/moonstreamdb/alembic/env.py b/moonstreamdb/alembic/env.py index 2e7def32..6d9eb01f 100644 --- a/moonstreamdb/alembic/env.py +++ b/moonstreamdb/alembic/env.py @@ -1,7 +1,6 @@ from logging.config import fileConfig -from sqlalchemy import engine_from_config -from sqlalchemy import pool +from sqlalchemy import engine_from_config, pool from alembic import context @@ -26,18 +25,27 @@ target_metadata = MoonstreamBase.metadata # my_important_option = config.get_main_option("my_important_option") # ... etc. from moonstreamdb.models import ( - EthereumBlock, - EthereumTransaction, - EthereumLabel, - PolygonBlock, - PolygonTransaction, - PolygonLabel, - MumbaiBlock, - MumbaiTransaction, - MumbaiLabel, - ESDFunctionSignature, ESDEventSignature, + ESDFunctionSignature, + EthereumBlock, + EthereumLabel, + EthereumTransaction, + MumbaiBlock, + MumbaiLabel, + MumbaiTransaction, OpenSeaCrawlingState, + PolygonBlock, + PolygonLabel, + PolygonTransaction, + WyrmBlock, + WyrmLabel, + WyrmTransaction, + XDaiBlock, + XDaiLabel, + XDaiTransaction, + ZkSyncEraTestnetBlock, + ZkSyncEraTestnetLabel, + ZkSyncEraTestnetTransaction, ) @@ -55,6 +63,15 @@ def include_symbol(tablename, schema): ESDFunctionSignature.__tablename__, ESDEventSignature.__tablename__, OpenSeaCrawlingState.__tablename__, + WyrmBlock.__tablename__, + WyrmLabel.__tablename__, + WyrmTransaction.__tablename__, + XDaiBlock.__tablename__, + XDaiLabel.__tablename__, + XDaiTransaction.__tablename__, + ZkSyncEraTestnetBlock.__tablename__, + ZkSyncEraTestnetLabel.__tablename__, + ZkSyncEraTestnetTransaction.__tablename__, } diff --git a/moonstreamdb/alembic/versions/e4a796c0407d_zksync_era_model.py b/moonstreamdb/alembic/versions/e4a796c0407d_zksync_era_model.py new file mode 100644 index 00000000..283fbe4f --- /dev/null +++ b/moonstreamdb/alembic/versions/e4a796c0407d_zksync_era_model.py @@ -0,0 +1,122 @@ +"""ZkSync Era model + +Revision ID: e4a796c0407d +Revises: c413d5265f76 +Create Date: 2023-07-12 12:27:12.814892 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'e4a796c0407d' +down_revision = 'c413d5265f76' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('zksync_era_testnet_blocks', + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('difficulty', sa.BigInteger(), nullable=True), + sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True), + sa.Column('gas_limit', sa.BigInteger(), nullable=True), + sa.Column('gas_used', sa.BigInteger(), nullable=True), + sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True), + sa.Column('miner', sa.VARCHAR(length=256), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('uncles', sa.VARCHAR(length=256), nullable=True), + sa.Column('size', sa.Integer(), nullable=True), + sa.Column('state_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('timestamp', sa.BigInteger(), nullable=True), + sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True), + sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.Column('mix_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('sha3_uncles', sa.VARCHAR(length=256), nullable=True), + sa.Column('l1_batch_number', sa.BigInteger(), nullable=True), + sa.Column('l1_batch_timestamp', sa.BigInteger(), nullable=True), + sa.PrimaryKeyConstraint('block_number', name=op.f('pk_zksync_era_testnet_blocks')) + ) + op.create_index(op.f('ix_zksync_era_testnet_blocks_block_number'), 'zksync_era_testnet_blocks', ['block_number'], unique=True) + op.create_index(op.f('ix_zksync_era_testnet_blocks_hash'), 'zksync_era_testnet_blocks', ['hash'], unique=False) + op.create_index(op.f('ix_zksync_era_testnet_blocks_timestamp'), 'zksync_era_testnet_blocks', ['timestamp'], unique=False) + op.create_table('zksync_era_testnet_labels', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('label', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=True), + sa.Column('address', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('block_timestamp', sa.BigInteger(), nullable=True), + sa.Column('log_index', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_zksync_era_testnet_labels')), + sa.UniqueConstraint('id', name=op.f('uq_zksync_era_testnet_labels_id')) + ) + op.create_index(op.f('ix_zksync_era_testnet_labels_address'), 'zksync_era_testnet_labels', ['address'], unique=False) + op.create_index('ix_zksync_era_testnet_labels_address_block_number', 'zksync_era_testnet_labels', ['address', 'block_number'], unique=False) + op.create_index('ix_zksync_era_testnet_labels_address_block_timestamp', 'zksync_era_testnet_labels', ['address', 'block_timestamp'], unique=False) + op.create_index(op.f('ix_zksync_era_testnet_labels_block_number'), 'zksync_era_testnet_labels', ['block_number'], unique=False) + op.create_index(op.f('ix_zksync_era_testnet_labels_block_timestamp'), 'zksync_era_testnet_labels', ['block_timestamp'], unique=False) + op.create_index(op.f('ix_zksync_era_testnet_labels_label'), 'zksync_era_testnet_labels', ['label'], unique=False) + op.create_index(op.f('ix_zksync_era_testnet_labels_transaction_hash'), 'zksync_era_testnet_labels', ['transaction_hash'], unique=False) + op.create_table('zksync_era_testnet_transactions', + sa.Column('hash', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('from_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('to_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('input', sa.Text(), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_index', sa.BigInteger(), nullable=True), + sa.Column('transaction_type', sa.Integer(), nullable=True), + sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.Column('l1_batch_number', sa.BigInteger(), nullable=True), + sa.Column('l1_batch_tx_index', sa.BigInteger(), nullable=True), + sa.ForeignKeyConstraint(['block_number'], ['zksync_era_testnet_blocks.block_number'], name=op.f('fk_zksync_era_testnet_transactions_block_number_zksync_era_testnet_blocks'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('hash', name=op.f('pk_zksync_era_testnet_transactions')) + ) + op.create_index(op.f('ix_zksync_era_testnet_transactions_block_number'), 'zksync_era_testnet_transactions', ['block_number'], unique=False) + op.create_index(op.f('ix_zksync_era_testnet_transactions_from_address'), 'zksync_era_testnet_transactions', ['from_address'], unique=False) + op.create_index(op.f('ix_zksync_era_testnet_transactions_gas'), 'zksync_era_testnet_transactions', ['gas'], unique=False) + op.create_index(op.f('ix_zksync_era_testnet_transactions_gas_price'), 'zksync_era_testnet_transactions', ['gas_price'], unique=False) + op.create_index(op.f('ix_zksync_era_testnet_transactions_hash'), 'zksync_era_testnet_transactions', ['hash'], unique=True) + op.create_index(op.f('ix_zksync_era_testnet_transactions_to_address'), 'zksync_era_testnet_transactions', ['to_address'], unique=False) + op.create_index(op.f('ix_zksync_era_testnet_transactions_value'), 'zksync_era_testnet_transactions', ['value'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_zksync_era_testnet_transactions_value'), table_name='zksync_era_testnet_transactions') + op.drop_index(op.f('ix_zksync_era_testnet_transactions_to_address'), table_name='zksync_era_testnet_transactions') + op.drop_index(op.f('ix_zksync_era_testnet_transactions_hash'), table_name='zksync_era_testnet_transactions') + op.drop_index(op.f('ix_zksync_era_testnet_transactions_gas_price'), table_name='zksync_era_testnet_transactions') + op.drop_index(op.f('ix_zksync_era_testnet_transactions_gas'), table_name='zksync_era_testnet_transactions') + op.drop_index(op.f('ix_zksync_era_testnet_transactions_from_address'), table_name='zksync_era_testnet_transactions') + op.drop_index(op.f('ix_zksync_era_testnet_transactions_block_number'), table_name='zksync_era_testnet_transactions') + op.drop_table('zksync_era_testnet_transactions') + op.drop_index(op.f('ix_zksync_era_testnet_labels_transaction_hash'), table_name='zksync_era_testnet_labels') + op.drop_index(op.f('ix_zksync_era_testnet_labels_label'), table_name='zksync_era_testnet_labels') + op.drop_index(op.f('ix_zksync_era_testnet_labels_block_timestamp'), table_name='zksync_era_testnet_labels') + op.drop_index(op.f('ix_zksync_era_testnet_labels_block_number'), table_name='zksync_era_testnet_labels') + op.drop_index('ix_zksync_era_testnet_labels_address_block_timestamp', table_name='zksync_era_testnet_labels') + op.drop_index('ix_zksync_era_testnet_labels_address_block_number', table_name='zksync_era_testnet_labels') + op.drop_index(op.f('ix_zksync_era_testnet_labels_address'), table_name='zksync_era_testnet_labels') + op.drop_table('zksync_era_testnet_labels') + op.drop_index(op.f('ix_zksync_era_testnet_blocks_timestamp'), table_name='zksync_era_testnet_blocks') + op.drop_index(op.f('ix_zksync_era_testnet_blocks_hash'), table_name='zksync_era_testnet_blocks') + op.drop_index(op.f('ix_zksync_era_testnet_blocks_block_number'), table_name='zksync_era_testnet_blocks') + op.drop_table('zksync_era_testnet_blocks') + # ### end Alembic commands ### diff --git a/moonstreamdb/moonstreamdb/blockchain.py b/moonstreamdb/moonstreamdb/blockchain.py index d2909892..90935343 100644 --- a/moonstreamdb/moonstreamdb/blockchain.py +++ b/moonstreamdb/moonstreamdb/blockchain.py @@ -1,26 +1,27 @@ -from .db import yield_db_session, yield_db_session_ctx +from enum import Enum +from typing import Type, Union + from .models import ( EthereumBlock, EthereumLabel, EthereumTransaction, - PolygonBlock, - PolygonLabel, - PolygonTransaction, MumbaiBlock, MumbaiLabel, MumbaiTransaction, - XDaiBlock, - XDaiLabel, - XDaiTransaction, + PolygonBlock, + PolygonLabel, + PolygonTransaction, WyrmBlock, WyrmLabel, WyrmTransaction, + XDaiBlock, + XDaiLabel, + XDaiTransaction, + ZkSyncEraTestnetBlock, + ZkSyncEraTestnetLabel, + ZkSyncEraTestnetTransaction, ) -from enum import Enum - -from typing import Type, Union - class AvailableBlockchainType(Enum): ETHEREUM = "ethereum" @@ -28,17 +29,34 @@ class AvailableBlockchainType(Enum): MUMBAI = "mumbai" XDAI = "xdai" WYRM = "wyrm" + ZKSYNC_ERA_TESTNET = "zksync_era_testnet" def get_block_model( blockchain_type: AvailableBlockchainType, -) -> Type[Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock, WyrmBlock]]: +) -> Type[ + Union[ + EthereumBlock, + PolygonBlock, + MumbaiBlock, + XDaiBlock, + WyrmBlock, + ZkSyncEraTestnetBlock, + ] +]: """ Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm set proper blocks model. """ block_model: Type[ - Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock, WyrmBlock] + Union[ + EthereumBlock, + PolygonBlock, + MumbaiBlock, + XDaiBlock, + WyrmBlock, + ZkSyncEraTestnetBlock, + ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: block_model = EthereumBlock @@ -50,6 +68,8 @@ def get_block_model( block_model = XDaiBlock elif blockchain_type == AvailableBlockchainType.WYRM: block_model = WyrmBlock + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: + block_model = ZkSyncEraTestnetBlock else: raise Exception("Unsupported blockchain type provided") @@ -58,13 +78,29 @@ def get_block_model( def get_label_model( blockchain_type: AvailableBlockchainType, -) -> Type[Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel, WyrmLabel]]: +) -> Type[ + Union[ + EthereumLabel, + PolygonLabel, + MumbaiLabel, + XDaiLabel, + WyrmLabel, + ZkSyncEraTestnetLabel, + ] +]: """ Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm set proper block label model. """ label_model: Type[ - Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel, WyrmLabel] + Union[ + EthereumLabel, + PolygonLabel, + MumbaiLabel, + XDaiLabel, + WyrmLabel, + ZkSyncEraTestnetLabel, + ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: label_model = EthereumLabel @@ -76,6 +112,8 @@ def get_label_model( label_model = XDaiLabel elif blockchain_type == AvailableBlockchainType.WYRM: label_model = WyrmLabel + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: + label_model = ZkSyncEraTestnetLabel else: raise Exception("Unsupported blockchain type provided") @@ -91,6 +129,7 @@ def get_transaction_model( MumbaiTransaction, XDaiTransaction, WyrmTransaction, + ZkSyncEraTestnetTransaction, ] ]: """ @@ -104,6 +143,7 @@ def get_transaction_model( MumbaiTransaction, XDaiTransaction, WyrmTransaction, + ZkSyncEraTestnetTransaction, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -116,6 +156,8 @@ def get_transaction_model( transaction_model = XDaiTransaction elif blockchain_type == AvailableBlockchainType.WYRM: transaction_model = WyrmTransaction + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: + transaction_model = ZkSyncEraTestnetTransaction else: raise Exception("Unsupported blockchain type provided") diff --git a/moonstreamdb/moonstreamdb/db.py b/moonstreamdb/moonstreamdb/db.py index d183aecd..c3730fe9 100644 --- a/moonstreamdb/moonstreamdb/db.py +++ b/moonstreamdb/moonstreamdb/db.py @@ -1,12 +1,12 @@ """ Moonstream database connection. """ -from contextlib import contextmanager import os +from contextlib import contextmanager from typing import Generator from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker, Session +from sqlalchemy.orm import Session, sessionmaker MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI") if MOONSTREAM_DB_URI is None: diff --git a/moonstreamdb/moonstreamdb/models.py b/moonstreamdb/moonstreamdb/models.py index fc0fdfdf..6819eb44 100644 --- a/moonstreamdb/moonstreamdb/models.py +++ b/moonstreamdb/moonstreamdb/models.py @@ -1,21 +1,21 @@ import uuid -from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import ( + VARCHAR, BigInteger, Column, DateTime, + ForeignKey, Index, Integer, - ForeignKey, MetaData, Numeric, Text, - VARCHAR, ) from sqlalchemy.dialects.postgresql import JSONB, UUID -from sqlalchemy.sql import expression from sqlalchemy.ext.compiler import compiles +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.sql import expression """ Naming conventions doc @@ -614,6 +614,137 @@ class WyrmLabel(Base): # type: ignore ) +class ZkSyncEraTestnetBlock(Base): # type: ignore + __tablename__ = "zksync_era_testnet_blocks" + + block_number = Column( + BigInteger, primary_key=True, unique=True, nullable=False, index=True + ) + difficulty = Column(BigInteger) + extra_data = Column(VARCHAR(128)) + gas_limit = Column(BigInteger) + gas_used = Column(BigInteger) + base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + hash = Column(VARCHAR(256), index=True) + logs_bloom = Column(VARCHAR(1024)) + miner = Column(VARCHAR(256)) + nonce = Column(VARCHAR(256)) + parent_hash = Column(VARCHAR(256)) + receipt_root = Column(VARCHAR(256)) + uncles = Column(VARCHAR(256)) + size = Column(Integer) + state_root = Column(VARCHAR(256)) + timestamp = Column(BigInteger, index=True) + total_difficulty = Column(VARCHAR(256)) + transactions_root = Column(VARCHAR(256)) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + mix_hash = Column(VARCHAR(256), nullable=True) + sha3_uncles = Column(VARCHAR(256), nullable=True) + + l1_batch_number = Column(BigInteger, nullable=True) + l1_batch_timestamp = Column(BigInteger, nullable=True) + + +class ZkSyncEraTestnetTransaction(Base): # type: ignore + __tablename__ = "zksync_era_testnet_transactions" + + hash = Column( + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True + ) + block_number = Column( + BigInteger, + ForeignKey("zksync_era_testnet_blocks.block_number", ondelete="CASCADE"), + nullable=False, + index=True, + ) + from_address = Column(VARCHAR(256), index=True) + to_address = Column(VARCHAR(256), index=True) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) + max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + input = Column(Text) + nonce = Column(VARCHAR(256)) + transaction_index = Column(BigInteger) + transaction_type = Column(Integer, nullable=True) + value = Column(Numeric(precision=78, scale=0), index=True) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + l1_batch_number = Column(BigInteger, nullable=True) + l1_batch_tx_index = Column(BigInteger, nullable=True) + + +class ZkSyncEraTestnetLabel(Base): # type: ignore + """ + Example of label_data: + { + "label": "ERC20", + "label_data": { + "name": "Uniswap", + "symbol": "UNI" + } + }, + { + "label": "Exchange" + "label_data": {...} + } + """ + + __tablename__ = "zksync_era_testnet_labels" + + __table_args__ = ( + Index( + "ix_zksync_era_testnet_labels_address_block_number", + "address", + "block_number", + unique=False, + ), + Index( + "ix_zksync_era_testnet_labels_address_block_timestamp", + "address", + "block_timestamp", + unique=False, + ), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + nullable=False, + ) + label = Column(VARCHAR(256), nullable=False, index=True) + block_number = Column( + BigInteger, + nullable=True, + index=True, + ) + address = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + transaction_hash = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + label_data = Column(JSONB, nullable=True) + block_timestamp = Column(BigInteger, index=True) + log_index = Column(Integer, nullable=True) + created_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + class ESDFunctionSignature(Base): # type: ignore """ Function signature from blockchain (Ethereum/Polygon) Signature Database. diff --git a/moonstreamdb/moonstreamdb/networks.py b/moonstreamdb/moonstreamdb/networks.py index 3f9668ee..b2d9fa8e 100644 --- a/moonstreamdb/moonstreamdb/networks.py +++ b/moonstreamdb/moonstreamdb/networks.py @@ -18,6 +18,9 @@ from .models import ( XDaiBlock, XDaiLabel, XDaiTransaction, + ZkSyncEraTestnetBlock, + ZkSyncEraTestnetLabel, + ZkSyncEraTestnetTransaction, ) @@ -27,6 +30,7 @@ class Network(Enum): mumbai = "mumbai" xdai = "xdai" wyrm = "wyrm" + zksync_era_testnet = "zksync_era_testnet" tx_raw_types = Union[ @@ -35,6 +39,7 @@ tx_raw_types = Union[ PolygonTransaction, WyrmTransaction, XDaiTransaction, + ZkSyncEraTestnetTransaction, ] MODELS: Dict[Network, Dict[str, Base]] = { @@ -63,4 +68,9 @@ MODELS: Dict[Network, Dict[str, Base]] = { "labels": WyrmLabel, "transactions": WyrmTransaction, }, + Network.zksync_era_testnet: { + "blocks": ZkSyncEraTestnetBlock, + "labels": ZkSyncEraTestnetLabel, + "transactions": ZkSyncEraTestnetTransaction, + }, } diff --git a/moonstreamdb/moonstreamdb/version.py b/moonstreamdb/moonstreamdb/version.py index f796c64b..b1acec14 100644 --- a/moonstreamdb/moonstreamdb/version.py +++ b/moonstreamdb/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.3.3" +MOONSTREAMDB_VERSION = "0.3.4"