mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #841 from moonstream-to/zksync-era-testnet-models 

ZkSync Era testnet model
pull/842/head
Sergei Sumarokov 2023-07-13 02:04:06 -07:00 zatwierdzone przez GitHub
rodzic 358aadcec1 a9186181d8
commit 1f6105afce
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
21 zmienionych plików z 451 dodań i 36 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

23
crawlers/deploy/deploy.bash
Wyświetl plik

@ -106,6 +106,11 @@ WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transaction
WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="wyrm-historical-crawl-events.service"
WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="wyrm-historical-crawl-events.timer"
# ZkSync Era testnet
ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service"
ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE="zksync-era-testnet-missing.service"
ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE="zksync-era-testnet-missing.service"
set -eu
echo
@ -499,3 +504,21 @@ cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/home/ubuntu/.c
cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}"
# ZkSync Era
echo
echo
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet block with transactions syncronizer service definition with ${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet missing service and timer with: ${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_MISSING_TIMER_FILE}"

11
crawlers/deploy/zksync-era-testnet-missing.service 100644
Wyświetl plik

@ -0,0 +1,11 @@
[Unit]
Description=Fill missing blocks at ZkSync Era testnet database
After=network.target
[Service]
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era_testnet -n
CPUWeight=50
SyslogIdentifier=zksync-era-testnet-missing

9
crawlers/deploy/zksync-era-testnet-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at ZkSync Era testnet database
[Timer]
OnBootSec=120s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

17
crawlers/deploy/zksync-era-testnet-synchronize.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=ZkSync Era testnet block with transactions synchronizer
StartLimitIntervalSec=300
StartLimitBurst=3
After=network.target
[Service]
Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2
CPUWeight=90
SyslogIdentifier=zksync-era-testnet-synchronize
[Install]
WantedBy=multi-user.target

11
crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -27,6 +27,7 @@ from .settings import (
MOONSTREAM_POLYGON_WEB3_PROVIDER_URI,
MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI,
NB_ACCESS_ID_HEADER,
NB_DATA_SOURCE_HEADER,
WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
@ -70,6 +71,8 @@ def connect(
web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.WYRM:
web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
else:
raise Exception("Wrong blockchain type provided for web3 URI")
@ -123,6 +126,11 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
)
if blockchain_type == AvailableBlockchainType.XDAI:
block_obj.author = block.author
if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
block_obj.mix_hash = block.get("mixHash", "")
block_obj.sha3_uncles = block.get("sha3Uncles", "")
block_obj.l1_batch_number = block.get("l1BatchNumber", None)
block_obj.l1_batch_timestamp = block.get("l1BatchTimestamp", None)
db_session.add(block_obj)
@ -152,6 +160,9 @@ def add_block_transactions(
transaction_type=int(tx["type"], 0) if tx.get("type") is not None else None,
value=tx.value,
)
if blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
tx_obj.l1_batch_number = tx.get("l1BatchNumber", None)
tx_obj.l1_batch_tx_index = tx.get("l1BatchTxIndex", None)
db_session.add(tx_obj)

2
crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py
Wyświetl plik

@ -130,6 +130,8 @@ def continuous_crawler(
network = Network.xdai
elif blockchain_type == AvailableBlockchainType.WYRM:
network = Network.wyrm
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
network = Network.zksync_era_testnet
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

3
crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py
Wyświetl plik

@ -35,6 +35,7 @@ class SubscriptionTypes(Enum):
MUMBAI_BLOCKCHAIN = "mumbai_smartcontract"
XDAI_BLOCKCHAIN = "xdai_smartcontract"
WYRM_BLOCKCHAIN = "wyrm_smartcontract"
ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract"
def abi_input_signature(input_abi: Dict[str, Any]) -> str:
@ -139,6 +140,8 @@ def blockchain_type_to_subscription_type(
return SubscriptionTypes.XDAI_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.WYRM:
return SubscriptionTypes.WYRM_BLOCKCHAIN
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
return SubscriptionTypes.ZKSYNC_ERA_TESTNET_BLOCKCHAIN
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

2
crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py
Wyświetl plik

@ -68,6 +68,8 @@ def function_call_crawler(
network = Network.xdai
elif blockchain_type == AvailableBlockchainType.WYRM:
network = Network.wyrm
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
network = Network.zksync_era_testnet
else:
raise ValueError(f"Unknown blockchain type: {blockchain_type}")

8
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -114,6 +114,14 @@ MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get(
if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "":
raise Exception(
"MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")
try:

3
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py
Wyświetl plik

@ -48,6 +48,7 @@ subscription_id_by_blockchain = {
"mumbai": "mumbai_smartcontract",
"xdai": "xdai_smartcontract",
"wyrm": "wyrm_smartcontract",
"zksync_era_testnet": "zksync_era_testnet_smartcontract",
}
blockchain_by_subscription_id = {
@ -56,11 +57,13 @@ blockchain_by_subscription_id = {
"mumbai_blockchain": "mumbai",
"xdai_blockchain": "xdai",
"wyrm_blockchain": "wyrm",
"zksync_era_testnet_blockchain": "zksync_era_testnet",
"ethereum_smartcontract": "ethereum",
"polygon_smartcontract": "polygon",
"mumbai_smartcontract": "mumbai",
"xdai_smartcontract": "xdai",
"wyrm_smartcontract": "wyrm",
"zksync_era_testnet_smartcontract": "zksync_era_testnet",
}

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.3.1"
MOONCRAWL_VERSION = "0.3.2"

1
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -25,6 +25,7 @@ export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://<connection_path_uri_to_pol
export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_mumbai_node>"
export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_node>"
export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_testnet_node>"
export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
# AWS environment variables

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -37,7 +37,7 @@ setup(
"bugout>=0.2.8",
"chardet",
"fastapi",
"moonstreamdb>=0.3.3",
"moonstreamdb>=0.3.4",
"moonstream>=0.1.1",
"moonstream-entity>=0.0.5",
"moonworm[moonstream]>=0.6.2",

3
moonstreamdb/.isort.cfg 100644
Wyświetl plik

@ -0,0 +1,3 @@
[settings]
profile = black
multi_line_output = 3

41
moonstreamdb/alembic/env.py
Wyświetl plik

@ -1,7 +1,6 @@
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from sqlalchemy import engine_from_config, pool
from alembic import context
@ -26,18 +25,27 @@ target_metadata = MoonstreamBase.metadata
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
from moonstreamdb.models import (
EthereumBlock,
EthereumTransaction,
EthereumLabel,
PolygonBlock,
PolygonTransaction,
PolygonLabel,
MumbaiBlock,
MumbaiTransaction,
MumbaiLabel,
ESDFunctionSignature,
ESDEventSignature,
ESDFunctionSignature,
EthereumBlock,
EthereumLabel,
EthereumTransaction,
MumbaiBlock,
MumbaiLabel,
MumbaiTransaction,
OpenSeaCrawlingState,
PolygonBlock,
PolygonLabel,
PolygonTransaction,
WyrmBlock,
WyrmLabel,
WyrmTransaction,
XDaiBlock,
XDaiLabel,
XDaiTransaction,
ZkSyncEraTestnetBlock,
ZkSyncEraTestnetLabel,
ZkSyncEraTestnetTransaction,
)
@ -55,6 +63,15 @@ def include_symbol(tablename, schema):
ESDFunctionSignature.__tablename__,
ESDEventSignature.__tablename__,
OpenSeaCrawlingState.__tablename__,
WyrmBlock.__tablename__,
WyrmLabel.__tablename__,
WyrmTransaction.__tablename__,
XDaiBlock.__tablename__,
XDaiLabel.__tablename__,
XDaiTransaction.__tablename__,
ZkSyncEraTestnetBlock.__tablename__,
ZkSyncEraTestnetLabel.__tablename__,
ZkSyncEraTestnetTransaction.__tablename__,
}

122
moonstreamdb/alembic/versions/e4a796c0407d_zksync_era_model.py 100644
Wyświetl plik

@ -0,0 +1,122 @@
"""ZkSync Era model
Revision ID: e4a796c0407d
Revises: c413d5265f76
Create Date: 2023-07-12 12:27:12.814892
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = 'e4a796c0407d'
down_revision = 'c413d5265f76'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('zksync_era_testnet_blocks',
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('difficulty', sa.BigInteger(), nullable=True),
sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True),
sa.Column('gas_limit', sa.BigInteger(), nullable=True),
sa.Column('gas_used', sa.BigInteger(), nullable=True),
sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True),
sa.Column('miner', sa.VARCHAR(length=256), nullable=True),
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('uncles', sa.VARCHAR(length=256), nullable=True),
sa.Column('size', sa.Integer(), nullable=True),
sa.Column('state_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('timestamp', sa.BigInteger(), nullable=True),
sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True),
sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.Column('mix_hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('sha3_uncles', sa.VARCHAR(length=256), nullable=True),
sa.Column('l1_batch_number', sa.BigInteger(), nullable=True),
sa.Column('l1_batch_timestamp', sa.BigInteger(), nullable=True),
sa.PrimaryKeyConstraint('block_number', name=op.f('pk_zksync_era_testnet_blocks'))
)
op.create_index(op.f('ix_zksync_era_testnet_blocks_block_number'), 'zksync_era_testnet_blocks', ['block_number'], unique=True)
op.create_index(op.f('ix_zksync_era_testnet_blocks_hash'), 'zksync_era_testnet_blocks', ['hash'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_blocks_timestamp'), 'zksync_era_testnet_blocks', ['timestamp'], unique=False)
op.create_table('zksync_era_testnet_labels',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('label', sa.VARCHAR(length=256), nullable=False),
sa.Column('block_number', sa.BigInteger(), nullable=True),
sa.Column('address', sa.VARCHAR(length=256), nullable=True),
sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True),
sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column('block_timestamp', sa.BigInteger(), nullable=True),
sa.Column('log_index', sa.Integer(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_zksync_era_testnet_labels')),
sa.UniqueConstraint('id', name=op.f('uq_zksync_era_testnet_labels_id'))
)
op.create_index(op.f('ix_zksync_era_testnet_labels_address'), 'zksync_era_testnet_labels', ['address'], unique=False)
op.create_index('ix_zksync_era_testnet_labels_address_block_number', 'zksync_era_testnet_labels', ['address', 'block_number'], unique=False)
op.create_index('ix_zksync_era_testnet_labels_address_block_timestamp', 'zksync_era_testnet_labels', ['address', 'block_timestamp'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_labels_block_number'), 'zksync_era_testnet_labels', ['block_number'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_labels_block_timestamp'), 'zksync_era_testnet_labels', ['block_timestamp'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_labels_label'), 'zksync_era_testnet_labels', ['label'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_labels_transaction_hash'), 'zksync_era_testnet_labels', ['transaction_hash'], unique=False)
op.create_table('zksync_era_testnet_transactions',
sa.Column('hash', sa.VARCHAR(length=256), nullable=False),
sa.Column('block_number', sa.BigInteger(), nullable=False),
sa.Column('from_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('to_address', sa.VARCHAR(length=256), nullable=True),
sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('input', sa.Text(), nullable=True),
sa.Column('nonce', sa.VARCHAR(length=256), nullable=True),
sa.Column('transaction_index', sa.BigInteger(), nullable=True),
sa.Column('transaction_type', sa.Integer(), nullable=True),
sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True),
sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False),
sa.Column('l1_batch_number', sa.BigInteger(), nullable=True),
sa.Column('l1_batch_tx_index', sa.BigInteger(), nullable=True),
sa.ForeignKeyConstraint(['block_number'], ['zksync_era_testnet_blocks.block_number'], name=op.f('fk_zksync_era_testnet_transactions_block_number_zksync_era_testnet_blocks'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('hash', name=op.f('pk_zksync_era_testnet_transactions'))
)
op.create_index(op.f('ix_zksync_era_testnet_transactions_block_number'), 'zksync_era_testnet_transactions', ['block_number'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_transactions_from_address'), 'zksync_era_testnet_transactions', ['from_address'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_transactions_gas'), 'zksync_era_testnet_transactions', ['gas'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_transactions_gas_price'), 'zksync_era_testnet_transactions', ['gas_price'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_transactions_hash'), 'zksync_era_testnet_transactions', ['hash'], unique=True)
op.create_index(op.f('ix_zksync_era_testnet_transactions_to_address'), 'zksync_era_testnet_transactions', ['to_address'], unique=False)
op.create_index(op.f('ix_zksync_era_testnet_transactions_value'), 'zksync_era_testnet_transactions', ['value'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_zksync_era_testnet_transactions_value'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_to_address'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_hash'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_gas_price'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_gas'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_from_address'), table_name='zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_transactions_block_number'), table_name='zksync_era_testnet_transactions')
op.drop_table('zksync_era_testnet_transactions')
op.drop_index(op.f('ix_zksync_era_testnet_labels_transaction_hash'), table_name='zksync_era_testnet_labels')
op.drop_index(op.f('ix_zksync_era_testnet_labels_label'), table_name='zksync_era_testnet_labels')
op.drop_index(op.f('ix_zksync_era_testnet_labels_block_timestamp'), table_name='zksync_era_testnet_labels')
op.drop_index(op.f('ix_zksync_era_testnet_labels_block_number'), table_name='zksync_era_testnet_labels')
op.drop_index('ix_zksync_era_testnet_labels_address_block_timestamp', table_name='zksync_era_testnet_labels')
op.drop_index('ix_zksync_era_testnet_labels_address_block_number', table_name='zksync_era_testnet_labels')
op.drop_index(op.f('ix_zksync_era_testnet_labels_address'), table_name='zksync_era_testnet_labels')
op.drop_table('zksync_era_testnet_labels')
op.drop_index(op.f('ix_zksync_era_testnet_blocks_timestamp'), table_name='zksync_era_testnet_blocks')
op.drop_index(op.f('ix_zksync_era_testnet_blocks_hash'), table_name='zksync_era_testnet_blocks')
op.drop_index(op.f('ix_zksync_era_testnet_blocks_block_number'), table_name='zksync_era_testnet_blocks')
op.drop_table('zksync_era_testnet_blocks')
# ### end Alembic commands ###

72
moonstreamdb/moonstreamdb/blockchain.py
Wyświetl plik

@ -1,26 +1,27 @@
from .db import yield_db_session, yield_db_session_ctx
from enum import Enum
from typing import Type, Union
from .models import (
EthereumBlock,
EthereumLabel,
EthereumTransaction,
PolygonBlock,
PolygonLabel,
PolygonTransaction,
MumbaiBlock,
MumbaiLabel,
MumbaiTransaction,
XDaiBlock,
XDaiLabel,
XDaiTransaction,
PolygonBlock,
PolygonLabel,
PolygonTransaction,
WyrmBlock,
WyrmLabel,
WyrmTransaction,
XDaiBlock,
XDaiLabel,
XDaiTransaction,
ZkSyncEraTestnetBlock,
ZkSyncEraTestnetLabel,
ZkSyncEraTestnetTransaction,
)
from enum import Enum
from typing import Type, Union
class AvailableBlockchainType(Enum):
ETHEREUM = "ethereum"
@ -28,17 +29,34 @@ class AvailableBlockchainType(Enum):
MUMBAI = "mumbai"
XDAI = "xdai"
WYRM = "wyrm"
ZKSYNC_ERA_TESTNET = "zksync_era_testnet"
def get_block_model(
blockchain_type: AvailableBlockchainType,
) -> Type[Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock, WyrmBlock]]:
) -> Type[
Union[
EthereumBlock,
PolygonBlock,
MumbaiBlock,
XDaiBlock,
WyrmBlock,
ZkSyncEraTestnetBlock,
]
]:
"""
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm
set proper blocks model.
"""
block_model: Type[
Union[EthereumBlock, PolygonBlock, MumbaiBlock, XDaiBlock, WyrmBlock]
Union[
EthereumBlock,
PolygonBlock,
MumbaiBlock,
XDaiBlock,
WyrmBlock,
ZkSyncEraTestnetBlock,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
block_model = EthereumBlock
@ -50,6 +68,8 @@ def get_block_model(
block_model = XDaiBlock
elif blockchain_type == AvailableBlockchainType.WYRM:
block_model = WyrmBlock
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
block_model = ZkSyncEraTestnetBlock
else:
raise Exception("Unsupported blockchain type provided")
@ -58,13 +78,29 @@ def get_block_model(
def get_label_model(
blockchain_type: AvailableBlockchainType,
) -> Type[Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel, WyrmLabel]]:
) -> Type[
Union[
EthereumLabel,
PolygonLabel,
MumbaiLabel,
XDaiLabel,
WyrmLabel,
ZkSyncEraTestnetLabel,
]
]:
"""
Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm
set proper block label model.
"""
label_model: Type[
Union[EthereumLabel, PolygonLabel, MumbaiLabel, XDaiLabel, WyrmLabel]
Union[
EthereumLabel,
PolygonLabel,
MumbaiLabel,
XDaiLabel,
WyrmLabel,
ZkSyncEraTestnetLabel,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
label_model = EthereumLabel
@ -76,6 +112,8 @@ def get_label_model(
label_model = XDaiLabel
elif blockchain_type == AvailableBlockchainType.WYRM:
label_model = WyrmLabel
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
label_model = ZkSyncEraTestnetLabel
else:
raise Exception("Unsupported blockchain type provided")
@ -91,6 +129,7 @@ def get_transaction_model(
MumbaiTransaction,
XDaiTransaction,
WyrmTransaction,
ZkSyncEraTestnetTransaction,
]
]:
"""
@ -104,6 +143,7 @@ def get_transaction_model(
MumbaiTransaction,
XDaiTransaction,
WyrmTransaction,
ZkSyncEraTestnetTransaction,
]
]
if blockchain_type == AvailableBlockchainType.ETHEREUM:
@ -116,6 +156,8 @@ def get_transaction_model(
transaction_model = XDaiTransaction
elif blockchain_type == AvailableBlockchainType.WYRM:
transaction_model = WyrmTransaction
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
transaction_model = ZkSyncEraTestnetTransaction
else:
raise Exception("Unsupported blockchain type provided")

4
moonstreamdb/moonstreamdb/db.py
Wyświetl plik

@ -1,12 +1,12 @@
"""
Moonstream database connection.
"""
from contextlib import contextmanager
import os
from contextlib import contextmanager
from typing import Generator
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
from sqlalchemy.orm import Session, sessionmaker
MOONSTREAM_DB_URI = os.environ.get("MOONSTREAM_DB_URI")
if MOONSTREAM_DB_URI is None:

139
moonstreamdb/moonstreamdb/models.py
Wyświetl plik

@ -1,21 +1,21 @@
import uuid
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import (
VARCHAR,
BigInteger,
Column,
DateTime,
ForeignKey,
Index,
Integer,
ForeignKey,
MetaData,
Numeric,
Text,
VARCHAR,
)
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.sql import expression
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql import expression
"""
Naming conventions doc
@ -614,6 +614,137 @@ class WyrmLabel(Base): # type: ignore
)
class ZkSyncEraTestnetBlock(Base): # type: ignore
__tablename__ = "zksync_era_testnet_blocks"
block_number = Column(
BigInteger, primary_key=True, unique=True, nullable=False, index=True
)
difficulty = Column(BigInteger)
extra_data = Column(VARCHAR(128))
gas_limit = Column(BigInteger)
gas_used = Column(BigInteger)
base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
hash = Column(VARCHAR(256), index=True)
logs_bloom = Column(VARCHAR(1024))
miner = Column(VARCHAR(256))
nonce = Column(VARCHAR(256))
parent_hash = Column(VARCHAR(256))
receipt_root = Column(VARCHAR(256))
uncles = Column(VARCHAR(256))
size = Column(Integer)
state_root = Column(VARCHAR(256))
timestamp = Column(BigInteger, index=True)
total_difficulty = Column(VARCHAR(256))
transactions_root = Column(VARCHAR(256))
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
mix_hash = Column(VARCHAR(256), nullable=True)
sha3_uncles = Column(VARCHAR(256), nullable=True)
l1_batch_number = Column(BigInteger, nullable=True)
l1_batch_timestamp = Column(BigInteger, nullable=True)
class ZkSyncEraTestnetTransaction(Base): # type: ignore
__tablename__ = "zksync_era_testnet_transactions"
hash = Column(
VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True
)
block_number = Column(
BigInteger,
ForeignKey("zksync_era_testnet_blocks.block_number", ondelete="CASCADE"),
nullable=False,
index=True,
)
from_address = Column(VARCHAR(256), index=True)
to_address = Column(VARCHAR(256), index=True)
gas = Column(Numeric(precision=78, scale=0), index=True)
gas_price = Column(Numeric(precision=78, scale=0), index=True)
max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True)
input = Column(Text)
nonce = Column(VARCHAR(256))
transaction_index = Column(BigInteger)
transaction_type = Column(Integer, nullable=True)
value = Column(Numeric(precision=78, scale=0), index=True)
indexed_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
l1_batch_number = Column(BigInteger, nullable=True)
l1_batch_tx_index = Column(BigInteger, nullable=True)
class ZkSyncEraTestnetLabel(Base): # type: ignore
"""
Example of label_data:
{
"label": "ERC20",
"label_data": {
"name": "Uniswap",
"symbol": "UNI"
}
},
{
"label": "Exchange"
"label_data": {...}
}
"""
__tablename__ = "zksync_era_testnet_labels"
__table_args__ = (
Index(
"ix_zksync_era_testnet_labels_address_block_number",
"address",
"block_number",
unique=False,
),
Index(
"ix_zksync_era_testnet_labels_address_block_timestamp",
"address",
"block_timestamp",
unique=False,
),
)
id = Column(
UUID(as_uuid=True),
primary_key=True,
default=uuid.uuid4,
unique=True,
nullable=False,
)
label = Column(VARCHAR(256), nullable=False, index=True)
block_number = Column(
BigInteger,
nullable=True,
index=True,
)
address = Column(
VARCHAR(256),
nullable=True,
index=True,
)
transaction_hash = Column(
VARCHAR(256),
nullable=True,
index=True,
)
label_data = Column(JSONB, nullable=True)
block_timestamp = Column(BigInteger, index=True)
log_index = Column(Integer, nullable=True)
created_at = Column(
DateTime(timezone=True), server_default=utcnow(), nullable=False
)
class ESDFunctionSignature(Base): # type: ignore
"""
Function signature from blockchain (Ethereum/Polygon) Signature Database.

10
moonstreamdb/moonstreamdb/networks.py
Wyświetl plik

@ -18,6 +18,9 @@ from .models import (
XDaiBlock,
XDaiLabel,
XDaiTransaction,
ZkSyncEraTestnetBlock,
ZkSyncEraTestnetLabel,
ZkSyncEraTestnetTransaction,
)
@ -27,6 +30,7 @@ class Network(Enum):
mumbai = "mumbai"
xdai = "xdai"
wyrm = "wyrm"
zksync_era_testnet = "zksync_era_testnet"
tx_raw_types = Union[
@ -35,6 +39,7 @@ tx_raw_types = Union[
PolygonTransaction,
WyrmTransaction,
XDaiTransaction,
ZkSyncEraTestnetTransaction,
]
MODELS: Dict[Network, Dict[str, Base]] = {
@ -63,4 +68,9 @@ MODELS: Dict[Network, Dict[str, Base]] = {
"labels": WyrmLabel,
"transactions": WyrmTransaction,
},
Network.zksync_era_testnet: {
"blocks": ZkSyncEraTestnetBlock,
"labels": ZkSyncEraTestnetLabel,
"transactions": ZkSyncEraTestnetTransaction,
},
}

2
moonstreamdb/moonstreamdb/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream database version.
"""
MOONSTREAMDB_VERSION = "0.3.3"
MOONSTREAMDB_VERSION = "0.3.4"