From c9ef01b331f5dde175d77c07b7ec6c8262c12efb Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 21 Feb 2024 03:27:06 +0200 Subject: [PATCH 1/3] Add Xai chain. --- crawlers/deploy/deploy.bash | 33 +++ crawlers/deploy/xai-missing.service | 11 + crawlers/deploy/xai-missing.timer | 9 + crawlers/deploy/xai-moonworm-crawler.service | 17 ++ crawlers/deploy/xai-synchronize.service | 17 ++ crawlers/mooncrawl/mooncrawl/blockchain.py | 13 ++ .../moonworm_crawler/continuous_crawler.py | 3 +- .../mooncrawl/moonworm_crawler/crawler.py | 3 + .../moonworm_crawler/function_call_crawler.py | 2 + .../moonworm_crawler/historical_crawler.py | 2 + crawlers/mooncrawl/mooncrawl/settings.py | 7 + .../mooncrawl/stats_worker/dashboard.py | 3 + crawlers/mooncrawl/mooncrawl/version.py | 2 +- crawlers/mooncrawl/sample.env | 1 + crawlers/mooncrawl/setup.py | 2 +- moonstreamdb/alembic/env.py | 6 + .../a99b97acc39e_xai_blockchain_support.py | 192 ++++++++++++++++++ moonstreamdb/moonstreamdb/blockchain.py | 22 +- moonstreamdb/moonstreamdb/models.py | 100 +++++++++ moonstreamdb/moonstreamdb/networks.py | 10 + moonstreamdb/moonstreamdb/version.py | 2 +- 21 files changed, 450 insertions(+), 7 deletions(-) create mode 100644 crawlers/deploy/xai-missing.service create mode 100644 crawlers/deploy/xai-missing.timer create mode 100644 crawlers/deploy/xai-moonworm-crawler.service create mode 100644 crawlers/deploy/xai-synchronize.service create mode 100644 moonstreamdb/alembic/versions/a99b97acc39e_xai_blockchain_support.py diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index deb23162..0c6990b7 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -134,6 +134,12 @@ ARBITRUM_SEPOLIA_MISSING_TIMER_FILE="arbitrum-sepolia-missing.timer" ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-sepolia-moonworm-crawler.service" ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE="arbitrum-sepolia-synchronize.service" +# Xai +XAI_MISSING_SERVICE_FILE="xai-missing.service" +XAI_MISSING_TIMER_FILE="xai-missing.timer" +XAI_MOONWORM_CRAWLER_SERVICE_FILE="xai-moonworm-crawler.service" +XAI_SYNCHRONIZE_SERVICE="xai-synchronize.service" + set -eu echo @@ -650,3 +656,30 @@ cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubun XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" + + +# Xai +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Xai block with transactions syncronizer service definition with ${XAI_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${XAI_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${XAI_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${XAI_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Xai missing service and timer with: ${XAI_MISSING_SERVICE_FILE}, ${XAI_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${XAI_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${XAI_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${XAI_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_MISSING_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Xai moonworm crawler service definition with ${XAI_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${XAI_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${XAI_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBI \ No newline at end of file diff --git a/crawlers/deploy/xai-missing.service b/crawlers/deploy/xai-missing.service new file mode 100644 index 00000000..8aee0f34 --- /dev/null +++ b/crawlers/deploy/xai-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at Xai database +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain xai -n +CPUWeight=50 +SyslogIdentifier=xai-missing \ No newline at end of file diff --git a/crawlers/deploy/xai-missing.timer b/crawlers/deploy/xai-missing.timer new file mode 100644 index 00000000..52f95a92 --- /dev/null +++ b/crawlers/deploy/xai-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at Xai database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/xai-moonworm-crawler.service b/crawlers/deploy/xai-moonworm-crawler.service new file mode 100644 index 00000000..95d49d12 --- /dev/null +++ b/crawlers/deploy/xai-moonworm-crawler.service @@ -0,0 +1,17 @@ +[Unit] +Description=Xai moonworm crawler +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b xai --confirmations 10 --min-blocks-batch 20 +CPUWeight=70 +SyslogIdentifier=xai-moonworm-crawler + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/xai-synchronize.service b/crawlers/deploy/xai-synchronize.service new file mode 100644 index 00000000..91c4eb70 --- /dev/null +++ b/crawlers/deploy/xai-synchronize.service @@ -0,0 +1,17 @@ +[Unit] +Description=Xai block with transactions synchronizer +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain xai -c 10 -j 2 +CPUWeight=90 +SyslogIdentifier=xai-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index a9e353aa..596cc14f 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -28,6 +28,7 @@ from .settings import ( MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI, MOONSTREAM_POLYGON_WEB3_PROVIDER_URI, MOONSTREAM_WYRM_WEB3_PROVIDER_URI, + MOONSTREAM_XAI_WEB3_PROVIDER_URI, MOONSTREAM_XDAI_WEB3_PROVIDER_URI, MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI, MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI, @@ -82,6 +83,8 @@ def connect( web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: web3_uri = MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI + elif web3_uri == AvailableBlockchainType.XAI: + web3_uri = MOONSTREAM_XAI_WEB3_PROVIDER_URI else: raise Exception("Wrong blockchain type provided for web3 URI") @@ -176,6 +179,13 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) block_obj.send_root = block.get("sendRoot", "") block_obj.mix_hash = block.get("mixHash", "") + if blockchain_type == AvailableBlockchainType.XAI: + block_obj.sha3_uncles = block.get("sha3Uncles", "") + block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber")) + block_obj.send_count = hex_to_int(block.get("sendCount")) + block_obj.send_root = block.get("sendRoot", "") + block_obj.mix_hash = block.get("mixHash", "") + db_session.add(block_obj) @@ -224,6 +234,9 @@ def add_block_transactions( if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: tx_obj.y_parity = hex_to_int(tx.get("yParity")) + if blockchain_type == AvailableBlockchainType.XAI: + tx_obj.y_parity = hex_to_int(tx.get("yParity")) + db_session.add(tx_obj) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index ee3d96a7..6c3a811f 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -138,7 +138,8 @@ def continuous_crawler( network = Network.arbitrum_nova elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: network = Network.arbitrum_sepolia - + elif blockchain_type == AvailableBlockchainType.XAI: + network = Network.xai else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index e4b6c558..27ce102b 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -39,6 +39,7 @@ class SubscriptionTypes(Enum): ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract" ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract" ARBITRUM_SEPOLIA_BLOCKCHAIN = "arbitrum_sepolia_smartcontract" + XAI_BLOCKCHAIN = "xai_smartcontract" def abi_input_signature(input_abi: Dict[str, Any]) -> str: @@ -151,6 +152,8 @@ def blockchain_type_to_subscription_type( return SubscriptionTypes.ARBITRUM_NOVA_BLOCKCHAIN elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: return SubscriptionTypes.ARBITRUM_SEPOLIA_BLOCKCHAIN + elif blockchain_type == AvailableBlockchainType.XAI: + return SubscriptionTypes.XAI_BLOCKCHAIN else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py index 55a1bc9a..8a71f6ac 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py @@ -76,6 +76,8 @@ def function_call_crawler( network = Network.arbitrum_nova elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: network = Network.arbitrum_sepolia + elif blockchain_type == AvailableBlockchainType.XAI: + network = Network.xai else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py index 323031ff..4091ae90 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py @@ -69,6 +69,8 @@ def historical_crawler( network = Network.arbitrum_nova elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: network = Network.arbitrum_sepolia + elif blockchain_type == AvailableBlockchainType.XAI: + network = Network.xai else: raise Exception("Unsupported blockchain type provided") diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index d48628ce..4596c325 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -138,6 +138,13 @@ if MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI == "": "MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI env variable is not set" ) +MOONSTREAM_XAI_WEB3_PROVIDER_URI = os.environ.get( + "MOONSTREAM_XAI_WEB3_PROVIDER_URI", "" +) +if MOONSTREAM_XAI_WEB3_PROVIDER_URI == "": + raise Exception("MOONSTREAM_XAI_WEB3_PROVIDER_URI env variable is not set") + + MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") try: diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 161bd343..10303952 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -58,6 +58,7 @@ subscription_id_by_blockchain = { "zksync_era": "zksync_era_smartcontract", "arbitrum_nova": "arbitrum_nova_smartcontract", "arbitrum_sepolia": "arbitrum_sepolia_smartcontract", + "xai": "xai_smartcontract", } blockchain_by_subscription_id = { @@ -69,6 +70,7 @@ blockchain_by_subscription_id = { "zksync_era_testnet_blockchain": "zksync_era_testnet", "arbitrum_nova_blockchain": "arbitrum_nova", "arbitrum_sepolia_blockchain": "arbitrum_sepolia", + "xai_blockchain": "xai", "ethereum_smartcontract": "ethereum", "polygon_smartcontract": "polygon", "mumbai_smartcontract": "mumbai", @@ -78,6 +80,7 @@ blockchain_by_subscription_id = { "zksync_era_smartcontract": "zksync_era", "arbitrum_nova_smartcontract": "arbitrum_nova", "arbitrum_sepolia_smartcontract": "arbitrum_sepolia", + "xai_smartcontract": "xai", } diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index a8f1c39e..8eb58150 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.3.7" +MOONCRAWL_VERSION = "0.3.8" diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 6f7498bf..96dd1ef3 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -30,6 +30,7 @@ export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://=0.2.13", "chardet", "fastapi", - "moonstreamdb>=0.3.7", + "moonstreamdb>=0.3.8", "moonstream>=0.1.1", "moonworm[moonstream]>=0.6.2", "humbug", diff --git a/moonstreamdb/alembic/env.py b/moonstreamdb/alembic/env.py index 532f4763..d2c88104 100644 --- a/moonstreamdb/alembic/env.py +++ b/moonstreamdb/alembic/env.py @@ -52,6 +52,9 @@ from moonstreamdb.models import ( ArbitrumSepoliaBlock, ArbitrumSepoliaTransaction, ArbitrumSepoliaLabel, + XaiBlock, + XaiLabel, + XaiTransaction, ) @@ -84,6 +87,9 @@ def include_symbol(tablename, schema): ArbitrumSepoliaBlock.__tablename__, ArbitrumSepoliaTransaction.__tablename__, ArbitrumSepoliaLabel.__tablename__, + XaiBlock.__tablename__, + XaiLabel.__tablename__, + XaiTransaction.__tablename__, } diff --git a/moonstreamdb/alembic/versions/a99b97acc39e_xai_blockchain_support.py b/moonstreamdb/alembic/versions/a99b97acc39e_xai_blockchain_support.py new file mode 100644 index 00000000..139f18a1 --- /dev/null +++ b/moonstreamdb/alembic/versions/a99b97acc39e_xai_blockchain_support.py @@ -0,0 +1,192 @@ +"""Xai blockchain support + +Revision ID: a99b97acc39e +Revises: b38b75b11247 +Create Date: 2024-02-21 03:24:23.327484 + +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "a99b97acc39e" +down_revision = "b38b75b11247" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "xai_blocks", + sa.Column("block_number", sa.BigInteger(), nullable=False), + sa.Column("difficulty", sa.BigInteger(), nullable=True), + sa.Column("extra_data", sa.VARCHAR(length=128), nullable=True), + sa.Column("gas_limit", sa.BigInteger(), nullable=True), + sa.Column("gas_used", sa.BigInteger(), nullable=True), + sa.Column("base_fee_per_gas", sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column("hash", sa.VARCHAR(length=256), nullable=True), + sa.Column("logs_bloom", sa.VARCHAR(length=1024), nullable=True), + sa.Column("miner", sa.VARCHAR(length=256), nullable=True), + sa.Column("nonce", sa.VARCHAR(length=256), nullable=True), + sa.Column("parent_hash", sa.VARCHAR(length=256), nullable=True), + sa.Column("receipt_root", sa.VARCHAR(length=256), nullable=True), + sa.Column("uncles", sa.VARCHAR(length=256), nullable=True), + sa.Column("size", sa.Integer(), nullable=True), + sa.Column("state_root", sa.VARCHAR(length=256), nullable=True), + sa.Column("timestamp", sa.BigInteger(), nullable=True), + sa.Column("total_difficulty", sa.VARCHAR(length=256), nullable=True), + sa.Column("transactions_root", sa.VARCHAR(length=256), nullable=True), + sa.Column( + "indexed_at", + sa.DateTime(timezone=True), + server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), + nullable=False, + ), + sa.Column("sha3_uncles", sa.VARCHAR(length=256), nullable=True), + sa.Column("l1_block_number", sa.BigInteger(), nullable=True), + sa.Column("send_count", sa.BigInteger(), nullable=True), + sa.Column("send_root", sa.VARCHAR(length=256), nullable=True), + sa.Column("mix_hash", sa.VARCHAR(length=256), nullable=True), + sa.PrimaryKeyConstraint("block_number", name=op.f("pk_xai_blocks")), + ) + op.create_index( + op.f("ix_xai_blocks_block_number"), "xai_blocks", ["block_number"], unique=True + ) + op.create_index(op.f("ix_xai_blocks_hash"), "xai_blocks", ["hash"], unique=False) + op.create_index( + op.f("ix_xai_blocks_timestamp"), "xai_blocks", ["timestamp"], unique=False + ) + op.create_table( + "xai_labels", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("label", sa.VARCHAR(length=256), nullable=False), + sa.Column("block_number", sa.BigInteger(), nullable=True), + sa.Column("address", sa.VARCHAR(length=256), nullable=True), + sa.Column("transaction_hash", sa.VARCHAR(length=256), nullable=True), + sa.Column("label_data", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("block_timestamp", sa.BigInteger(), nullable=True), + sa.Column("log_index", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_xai_labels")), + sa.UniqueConstraint("id", name=op.f("uq_xai_labels_id")), + ) + op.create_index( + op.f("ix_xai_labels_address"), "xai_labels", ["address"], unique=False + ) + op.create_index( + op.f("ix_xai_labels_block_number"), "xai_labels", ["block_number"], unique=False + ) + op.create_index( + op.f("ix_xai_labels_block_timestamp"), + "xai_labels", + ["block_timestamp"], + unique=False, + ) + op.create_index(op.f("ix_xai_labels_label"), "xai_labels", ["label"], unique=False) + op.create_index( + op.f("ix_xai_labels_transaction_hash"), + "xai_labels", + ["transaction_hash"], + unique=False, + ) + op.create_table( + "xai_transactions", + sa.Column("hash", sa.VARCHAR(length=256), nullable=False), + sa.Column("block_number", sa.BigInteger(), nullable=False), + sa.Column("from_address", sa.VARCHAR(length=256), nullable=True), + sa.Column("to_address", sa.VARCHAR(length=256), nullable=True), + sa.Column("gas", sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column("gas_price", sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column("max_fee_per_gas", sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column( + "max_priority_fee_per_gas", sa.Numeric(precision=78, scale=0), nullable=True + ), + sa.Column("input", sa.Text(), nullable=True), + sa.Column("nonce", sa.VARCHAR(length=256), nullable=True), + sa.Column("transaction_index", sa.BigInteger(), nullable=True), + sa.Column("transaction_type", sa.Integer(), nullable=True), + sa.Column("value", sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column( + "indexed_at", + sa.DateTime(timezone=True), + server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), + nullable=False, + ), + sa.Column("y_parity", sa.BigInteger(), nullable=True), + sa.ForeignKeyConstraint( + ["block_number"], + ["xai_blocks.block_number"], + name=op.f("fk_xai_transactions_block_number_xai_blocks"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("hash", name=op.f("pk_xai_transactions")), + ) + op.create_index( + op.f("ix_xai_transactions_block_number"), + "xai_transactions", + ["block_number"], + unique=False, + ) + op.create_index( + op.f("ix_xai_transactions_from_address"), + "xai_transactions", + ["from_address"], + unique=False, + ) + op.create_index( + op.f("ix_xai_transactions_gas"), "xai_transactions", ["gas"], unique=False + ) + op.create_index( + op.f("ix_xai_transactions_gas_price"), + "xai_transactions", + ["gas_price"], + unique=False, + ) + op.create_index( + op.f("ix_xai_transactions_hash"), "xai_transactions", ["hash"], unique=True + ) + op.create_index( + op.f("ix_xai_transactions_to_address"), + "xai_transactions", + ["to_address"], + unique=False, + ) + op.create_index( + op.f("ix_xai_transactions_value"), "xai_transactions", ["value"], unique=False + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_xai_transactions_value"), table_name="xai_transactions") + op.drop_index(op.f("ix_xai_transactions_to_address"), table_name="xai_transactions") + op.drop_index(op.f("ix_xai_transactions_hash"), table_name="xai_transactions") + op.drop_index(op.f("ix_xai_transactions_gas_price"), table_name="xai_transactions") + op.drop_index(op.f("ix_xai_transactions_gas"), table_name="xai_transactions") + op.drop_index( + op.f("ix_xai_transactions_from_address"), table_name="xai_transactions" + ) + op.drop_index( + op.f("ix_xai_transactions_block_number"), table_name="xai_transactions" + ) + op.drop_table("xai_transactions") + op.drop_index(op.f("ix_xai_labels_transaction_hash"), table_name="xai_labels") + op.drop_index(op.f("ix_xai_labels_label"), table_name="xai_labels") + op.drop_index(op.f("ix_xai_labels_block_timestamp"), table_name="xai_labels") + op.drop_index(op.f("ix_xai_labels_block_number"), table_name="xai_labels") + op.drop_index(op.f("ix_xai_labels_address"), table_name="xai_labels") + op.drop_table("xai_labels") + op.drop_index(op.f("ix_xai_blocks_timestamp"), table_name="xai_blocks") + op.drop_index(op.f("ix_xai_blocks_hash"), table_name="xai_blocks") + op.drop_index(op.f("ix_xai_blocks_block_number"), table_name="xai_blocks") + op.drop_table("xai_blocks") + # ### end Alembic commands ### diff --git a/moonstreamdb/moonstreamdb/blockchain.py b/moonstreamdb/moonstreamdb/blockchain.py index 4c8f21c2..af73e3eb 100644 --- a/moonstreamdb/moonstreamdb/blockchain.py +++ b/moonstreamdb/moonstreamdb/blockchain.py @@ -29,6 +29,9 @@ from .models import ( ArbitrumSepoliaBlock, ArbitrumSepoliaTransaction, ArbitrumSepoliaLabel, + XaiBlock, + XaiLabel, + XaiTransaction, ) @@ -42,6 +45,7 @@ class AvailableBlockchainType(Enum): ZKSYNC_ERA = "zksync_era" ARBITRUM_NOVA = "arbitrum_nova" ARBITRUM_SEPOLIA = "arbitrum_sepolia" + XAI = "xai" def get_block_model( @@ -57,10 +61,11 @@ def get_block_model( ZkSyncEraBlock, ArbitrumNovaBlock, ArbitrumSepoliaBlock, + XaiBlock, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock, ArbitrumSepoliaBlock + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock, ArbitrumSepoliaBlock, XaiBlock set proper blocks model. """ block_model: Type[ @@ -74,6 +79,7 @@ def get_block_model( ZkSyncEraBlock, ArbitrumNovaBlock, ArbitrumSepoliaBlock, + XaiBlock, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -94,6 +100,8 @@ def get_block_model( block_model = ArbitrumNovaBlock elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: block_model = ArbitrumSepoliaBlock + elif blockchain_type == AvailableBlockchainType.XAI: + block_model = XaiBlock else: raise Exception("Unsupported blockchain type provided") @@ -113,10 +121,11 @@ def get_label_model( ZkSyncEraLabel, ArbitrumNovaLabel, ArbitrumSepoliaLabel, + XaiLabel, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel, ArbitrumSepoliaLabel + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel, ArbitrumSepoliaLabel, XaiLabel set proper block label model. """ label_model: Type[ @@ -130,6 +139,7 @@ def get_label_model( ZkSyncEraLabel, ArbitrumNovaLabel, ArbitrumSepoliaLabel, + XaiLabel, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -150,6 +160,8 @@ def get_label_model( label_model = ArbitrumNovaLabel elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: label_model = ArbitrumSepoliaLabel + elif blockchain_type == AvailableBlockchainType.XAI: + label_model = XaiLabel else: raise Exception("Unsupported blockchain type provided") @@ -169,10 +181,11 @@ def get_transaction_model( ZkSyncEraTransaction, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, + XaiTransaction, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, XaiTransaction set proper block transactions model. """ transaction_model: Type[ @@ -186,6 +199,7 @@ def get_transaction_model( ZkSyncEraTransaction, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, + XaiTransaction, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -206,6 +220,8 @@ def get_transaction_model( transaction_model = ArbitrumNovaTransaction elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: transaction_model = ArbitrumSepoliaTransaction + elif blockchain_type == AvailableBlockchainType.XAI: + transaction_model = XaiTransaction else: raise Exception("Unsupported blockchain type provided") diff --git a/moonstreamdb/moonstreamdb/models.py b/moonstreamdb/moonstreamdb/models.py index 602a3f8b..df223d35 100644 --- a/moonstreamdb/moonstreamdb/models.py +++ b/moonstreamdb/moonstreamdb/models.py @@ -1001,6 +1001,106 @@ class ArbitrumSepoliaLabel(Base): # type: ignore ) +class XaiBlock(Base): # type: ignore + __tablename__ = "xai_blocks" + + block_number = Column( + BigInteger, primary_key=True, unique=True, nullable=False, index=True + ) + difficulty = Column(BigInteger) + extra_data = Column(VARCHAR(128)) + gas_limit = Column(BigInteger) + gas_used = Column(BigInteger) + base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + hash = Column(VARCHAR(256), index=True) + logs_bloom = Column(VARCHAR(1024)) + miner = Column(VARCHAR(256)) + nonce = Column(VARCHAR(256)) + parent_hash = Column(VARCHAR(256)) + receipt_root = Column(VARCHAR(256)) + uncles = Column(VARCHAR(256)) + size = Column(Integer) + state_root = Column(VARCHAR(256)) + timestamp = Column(BigInteger, index=True) + total_difficulty = Column(VARCHAR(256)) + transactions_root = Column(VARCHAR(256)) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + sha3_uncles = Column(VARCHAR(256), nullable=True) + l1_block_number = Column(BigInteger, nullable=True) + send_count = Column(BigInteger, nullable=True) + send_root = Column(VARCHAR(256), nullable=True) + mix_hash = Column(VARCHAR(256), nullable=True) + + +class XaiTransaction(Base): # type: ignore + __tablename__ = "xai_transactions" + + hash = Column( + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True + ) + block_number = Column( + BigInteger, + ForeignKey("xai_blocks.block_number", ondelete="CASCADE"), + nullable=False, + index=True, + ) + from_address = Column(VARCHAR(256), index=True) + to_address = Column(VARCHAR(256), index=True) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) + max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + input = Column(Text) + nonce = Column(VARCHAR(256)) + transaction_index = Column(BigInteger) + transaction_type = Column(Integer, nullable=True) + value = Column(Numeric(precision=78, scale=0), index=True) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + y_parity = Column(BigInteger, nullable=True) + + +class XaiLabel(Base): # type: ignore + __tablename__ = "xai_labels" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + nullable=False, + ) + label = Column(VARCHAR(256), nullable=False, index=True) + block_number = Column( + BigInteger, + nullable=True, + index=True, + ) + address = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + transaction_hash = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + label_data = Column(JSONB, nullable=True) + block_timestamp = Column(BigInteger, index=True) + log_index = Column(Integer, nullable=True) + created_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + class ESDFunctionSignature(Base): # type: ignore """ Function signature from blockchain (Ethereum/Polygon) Signature Database. diff --git a/moonstreamdb/moonstreamdb/networks.py b/moonstreamdb/moonstreamdb/networks.py index b293826c..30256ca5 100644 --- a/moonstreamdb/moonstreamdb/networks.py +++ b/moonstreamdb/moonstreamdb/networks.py @@ -30,6 +30,9 @@ from .models import ( ArbitrumSepoliaBlock, ArbitrumSepoliaTransaction, ArbitrumSepoliaLabel, + XaiBlock, + XaiLabel, + XaiTransaction, ) @@ -43,6 +46,7 @@ class Network(Enum): zksync_era = "zksync_era" arbitrum_nova = "arbitrum_nova" arbitrum_sepolia = "arbitrum_sepolia" + xai = "xai" tx_raw_types = Union[ @@ -55,6 +59,7 @@ tx_raw_types = Union[ ZkSyncEraTransaction, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, + XaiTransaction, ] MODELS: Dict[Network, Dict[str, Base]] = { @@ -103,4 +108,9 @@ MODELS: Dict[Network, Dict[str, Base]] = { "labels": ArbitrumSepoliaLabel, "transactions": ArbitrumSepoliaTransaction, }, + Network.xai: { + "blocks": XaiBlock, + "labels": XaiLabel, + "transactions": XaiTransaction, + }, } diff --git a/moonstreamdb/moonstreamdb/version.py b/moonstreamdb/moonstreamdb/version.py index 6fcd0f3d..73a7e3a0 100644 --- a/moonstreamdb/moonstreamdb/version.py +++ b/moonstreamdb/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.3.7" +MOONSTREAMDB_VERSION = "0.3.8" From 3a0aa8993f46f4e3c389ffea10a876bbc0b90f84 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 21 Feb 2024 04:04:22 +0200 Subject: [PATCH 2/3] fix if --- crawlers/mooncrawl/mooncrawl/blockchain.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index 596cc14f..0e1d5df3 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -63,7 +63,6 @@ def connect( "Content-Type": "application/json", } } - if web3_uri is None: if blockchain_type == AvailableBlockchainType.ETHEREUM: web3_uri = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI @@ -83,7 +82,7 @@ def connect( web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: web3_uri = MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI - elif web3_uri == AvailableBlockchainType.XAI: + elif blockchain_type == AvailableBlockchainType.XAI: web3_uri = MOONSTREAM_XAI_WEB3_PROVIDER_URI else: raise Exception("Wrong blockchain type provided for web3 URI") From 10f37621e062a4f7d19c0a1204a135449e88c0cb Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 21 Feb 2024 04:54:56 +0200 Subject: [PATCH 3/3] Add monitoring. --- crawlers/deploy/monitoring-crawlers.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/deploy/monitoring-crawlers.service b/crawlers/deploy/monitoring-crawlers.service index 92bb90a7..05e1f379 100644 --- a/crawlers/deploy/monitoring-crawlers.service +++ b/crawlers/deploy/monitoring-crawlers.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/ EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env -ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service +ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service -service xai-moonworm-crawler.service CPUWeight=90 SyslogIdentifier=monitoring-crawlers