From 41b817660a18aa86f375dbafe92f5dd04bf1208e Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 14 Mar 2024 00:09:54 +0200 Subject: [PATCH] Add xai testnet. --- crawlers/deploy/deploy.bash | 35 +++++- crawlers/deploy/monitoring-crawlers.service | 2 +- crawlers/deploy/xai-testnet-missing.service | 11 ++ crawlers/deploy/xai-testnet-missing.timer | 9 ++ .../xai-testnet-moonworm-crawler.service | 17 +++ .../deploy/xai-testnet-synchronize.service | 17 +++ crawlers/mooncrawl/mooncrawl/blockchain.py | 13 +++ .../moonworm_crawler/continuous_crawler.py | 2 + .../mooncrawl/moonworm_crawler/crawler.py | 3 + .../moonworm_crawler/function_call_crawler.py | 2 + .../moonworm_crawler/historical_crawler.py | 2 + crawlers/mooncrawl/mooncrawl/settings.py | 5 + .../mooncrawl/stats_worker/dashboard.py | 3 + crawlers/mooncrawl/mooncrawl/version.py | 2 +- crawlers/mooncrawl/sample.env | 1 + crawlers/mooncrawl/setup.py | 2 +- moonstreamdb/alembic/env.py | 6 ++ moonstreamdb/moonstreamdb/blockchain.py | 22 +++- moonstreamdb/moonstreamdb/models.py | 100 ++++++++++++++++++ moonstreamdb/moonstreamdb/networks.py | 10 ++ moonstreamdb/moonstreamdb/version.py | 2 +- 21 files changed, 258 insertions(+), 8 deletions(-) create mode 100644 crawlers/deploy/xai-testnet-missing.service create mode 100644 crawlers/deploy/xai-testnet-missing.timer create mode 100644 crawlers/deploy/xai-testnet-moonworm-crawler.service create mode 100644 crawlers/deploy/xai-testnet-synchronize.service diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 2ab07d37..6c96df55 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -146,6 +146,12 @@ XAI_MISSING_TIMER_FILE="xai-missing.timer" XAI_MOONWORM_CRAWLER_SERVICE_FILE="xai-moonworm-crawler.service" XAI_SYNCHRONIZE_SERVICE="xai-synchronize.service" +# Xai testnet +XAI_TESTNET_MISSING_SERVICE_FILE="xai-testnet-missing.service" +XAI_TESTNET_MISSING_TIMER_FILE="xai-testnet-missing.timer" +XAI_TESTNET_MOONWORM_CRAWLER_SERVICE_FILE="xai-testnet-moonworm-crawler.service" +XAI_TESTNET_SYNCHRONIZE_SERVICE="xai-testnet-synchronize.service" + set -eu echo @@ -715,4 +721,31 @@ echo -e "${PREFIX_INFO} Replacing existing Xai moonworm crawler service definiti chmod 644 "${SCRIPT_DIR}/${XAI_MOONWORM_CRAWLER_SERVICE_FILE}" cp "${SCRIPT_DIR}/${XAI_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_MOONWORM_CRAWLER_SERVICE_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_MOONWORM_CRAWLER_SERVICE_FILE}" \ No newline at end of file +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_MOONWORM_CRAWLER_SERVICE_FILE}" + + +# Xai testnet +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Xai testnet block with transactions syncronizer service definition with ${XAI_TESTNET_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${XAI_TESTNET_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${XAI_TESTNET_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${XAI_TESTNET_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_TESTNET_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Xai testnet missing service and timer with: ${XAI_TESTNET_MISSING_SERVICE_FILE}, ${XAI_TESTNET_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${XAI_TESTNET_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${XAI_TESTNET_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${XAI_TESTNET_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_TESTNET_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${XAI_TESTNET_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_TESTNET_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_TESTNET_MISSING_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Xai testnet moonworm crawler service definition with ${XAI_TESTNET_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${XAI_TESTNET_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${XAI_TESTNET_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${XAI_TESTNET_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XAI_TESTNET_MOONWORM_CRAWLER_SERVICE_FILE}" diff --git a/crawlers/deploy/monitoring-crawlers.service b/crawlers/deploy/monitoring-crawlers.service index 05e1f379..a6d2d1a8 100644 --- a/crawlers/deploy/monitoring-crawlers.service +++ b/crawlers/deploy/monitoring-crawlers.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/ EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env -ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service -service xai-moonworm-crawler.service +ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service -service xai-moonworm-crawler.service -service xai-testnet-moonworm-crawler.service CPUWeight=90 SyslogIdentifier=monitoring-crawlers diff --git a/crawlers/deploy/xai-testnet-missing.service b/crawlers/deploy/xai-testnet-missing.service new file mode 100644 index 00000000..2f88a1d7 --- /dev/null +++ b/crawlers/deploy/xai-testnet-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at Xai testnet database +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain xai_testnet -n +CPUWeight=50 +SyslogIdentifier=xai-testnet-missing \ No newline at end of file diff --git a/crawlers/deploy/xai-testnet-missing.timer b/crawlers/deploy/xai-testnet-missing.timer new file mode 100644 index 00000000..6ed3613c --- /dev/null +++ b/crawlers/deploy/xai-testnet-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at Xai testnet database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/xai-testnet-moonworm-crawler.service b/crawlers/deploy/xai-testnet-moonworm-crawler.service new file mode 100644 index 00000000..77468341 --- /dev/null +++ b/crawlers/deploy/xai-testnet-moonworm-crawler.service @@ -0,0 +1,17 @@ +[Unit] +Description=Xai testnet moonworm crawler +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b xai_testnet --confirmations 10 --min-blocks-batch 20 +CPUWeight=70 +SyslogIdentifier=xai-test-moonworm-crawler + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/xai-testnet-synchronize.service b/crawlers/deploy/xai-testnet-synchronize.service new file mode 100644 index 00000000..6629730e --- /dev/null +++ b/crawlers/deploy/xai-testnet-synchronize.service @@ -0,0 +1,17 @@ +[Unit] +Description=Xai testnet block with transactions synchronizer +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain xai_testnet -c 10 -j 2 +CPUWeight=90 +SyslogIdentifier=xai-testnet-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index 0e1d5df3..bd7876ac 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -29,6 +29,7 @@ from .settings import ( MOONSTREAM_POLYGON_WEB3_PROVIDER_URI, MOONSTREAM_WYRM_WEB3_PROVIDER_URI, MOONSTREAM_XAI_WEB3_PROVIDER_URI, + MOONSTREAM_XAI_TESTNET_WEB3_PROVIDER_URI, MOONSTREAM_XDAI_WEB3_PROVIDER_URI, MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI, MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI, @@ -84,6 +85,8 @@ def connect( web3_uri = MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI elif blockchain_type == AvailableBlockchainType.XAI: web3_uri = MOONSTREAM_XAI_WEB3_PROVIDER_URI + elif blockchain_type == AvailableBlockchainType.XAI_TESTNET: + web3_uri = MOONSTREAM_XAI_TESTNET_WEB3_PROVIDER_URI else: raise Exception("Wrong blockchain type provided for web3 URI") @@ -185,6 +188,13 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) block_obj.send_root = block.get("sendRoot", "") block_obj.mix_hash = block.get("mixHash", "") + if blockchain_type == AvailableBlockchainType.XAI_TESTNET: + block_obj.sha3_uncles = block.get("sha3Uncles", "") + block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber")) + block_obj.send_count = hex_to_int(block.get("sendCount")) + block_obj.send_root = block.get("sendRoot", "") + block_obj.mix_hash = block.get("mixHash", "") + db_session.add(block_obj) @@ -236,6 +246,9 @@ def add_block_transactions( if blockchain_type == AvailableBlockchainType.XAI: tx_obj.y_parity = hex_to_int(tx.get("yParity")) + if blockchain_type == AvailableBlockchainType.XAI_TESTNET: + tx_obj.y_parity = hex_to_int(tx.get("yParity")) + db_session.add(tx_obj) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index 6c3a811f..b80f299e 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -140,6 +140,8 @@ def continuous_crawler( network = Network.arbitrum_sepolia elif blockchain_type == AvailableBlockchainType.XAI: network = Network.xai + elif blockchain_type == AvailableBlockchainType.XAI_TESTNET: + network = Network.xai_testnet else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index 27ce102b..640b9211 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -40,6 +40,7 @@ class SubscriptionTypes(Enum): ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract" ARBITRUM_SEPOLIA_BLOCKCHAIN = "arbitrum_sepolia_smartcontract" XAI_BLOCKCHAIN = "xai_smartcontract" + XAI_TESTNET_BLOCKCHAIN = "xai_testnet_smartcontract" def abi_input_signature(input_abi: Dict[str, Any]) -> str: @@ -154,6 +155,8 @@ def blockchain_type_to_subscription_type( return SubscriptionTypes.ARBITRUM_SEPOLIA_BLOCKCHAIN elif blockchain_type == AvailableBlockchainType.XAI: return SubscriptionTypes.XAI_BLOCKCHAIN + elif blockchain_type == AvailableBlockchainType.XAI_TESTNET: + return SubscriptionTypes.XAI_TESTNET_BLOCKCHAIN else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py index 8a71f6ac..14bdd33e 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py @@ -78,6 +78,8 @@ def function_call_crawler( network = Network.arbitrum_sepolia elif blockchain_type == AvailableBlockchainType.XAI: network = Network.xai + elif blockchain_type == AvailableBlockchainType.XAI_TESTNET: + network = Network.xai_testnet else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py index 4091ae90..4a91f9fe 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py @@ -71,6 +71,8 @@ def historical_crawler( network = Network.arbitrum_sepolia elif blockchain_type == AvailableBlockchainType.XAI: network = Network.xai + elif blockchain_type == AvailableBlockchainType.XAI_TESTNET: + network = Network.xai_testnet else: raise Exception("Unsupported blockchain type provided") diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 4596c325..c5dccdd5 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -144,6 +144,11 @@ MOONSTREAM_XAI_WEB3_PROVIDER_URI = os.environ.get( if MOONSTREAM_XAI_WEB3_PROVIDER_URI == "": raise Exception("MOONSTREAM_XAI_WEB3_PROVIDER_URI env variable is not set") +MOONSTREAM_XAI_TESTNET_WEB3_PROVIDER_URI = os.environ.get( + "MOONSTREAM_XAI_TESTNET_WEB3_PROVIDER_URI", "" +) +if MOONSTREAM_XAI_TESTNET_WEB3_PROVIDER_URI == "": + raise Exception("MOONSTREAM_XAI_TESTNET_WEB3_PROVIDER_URI env variable is not set") MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 10303952..52ab496a 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -59,6 +59,7 @@ subscription_id_by_blockchain = { "arbitrum_nova": "arbitrum_nova_smartcontract", "arbitrum_sepolia": "arbitrum_sepolia_smartcontract", "xai": "xai_smartcontract", + "xai_testnet": "xai_testnet_smartcontract", } blockchain_by_subscription_id = { @@ -71,6 +72,7 @@ blockchain_by_subscription_id = { "arbitrum_nova_blockchain": "arbitrum_nova", "arbitrum_sepolia_blockchain": "arbitrum_sepolia", "xai_blockchain": "xai", + "xai_testnet_blockchain": "xai_testnet", "ethereum_smartcontract": "ethereum", "polygon_smartcontract": "polygon", "mumbai_smartcontract": "mumbai", @@ -81,6 +83,7 @@ blockchain_by_subscription_id = { "arbitrum_nova_smartcontract": "arbitrum_nova", "arbitrum_sepolia_smartcontract": "arbitrum_sepolia", "xai_smartcontract": "xai", + "xai_testnet_smartcontract": "xai_testnet", } diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index 8eb58150..e999d730 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.3.8" +MOONCRAWL_VERSION = "0.3.9" diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 96dd1ef3..0b7f7b13 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -31,6 +31,7 @@ export MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI="https://=0.2.13", "chardet", "fastapi", - "moonstreamdb>=0.3.8", + "moonstreamdb>=0.3.9", "moonstream>=0.1.1", "moonworm[moonstream]>=0.6.2", "humbug", diff --git a/moonstreamdb/alembic/env.py b/moonstreamdb/alembic/env.py index d2c88104..f92db4d5 100644 --- a/moonstreamdb/alembic/env.py +++ b/moonstreamdb/alembic/env.py @@ -55,6 +55,9 @@ from moonstreamdb.models import ( XaiBlock, XaiLabel, XaiTransaction, + XaiTestnetBlock, + XaiTestnetLabel, + XaiTestnetTransaction, ) @@ -90,6 +93,9 @@ def include_symbol(tablename, schema): XaiBlock.__tablename__, XaiLabel.__tablename__, XaiTransaction.__tablename__, + XaiTestnetBlock.__tablename__, + XaiTestnetLabel.__tablename__, + XaiTestnetTransaction.__tablename__, } diff --git a/moonstreamdb/moonstreamdb/blockchain.py b/moonstreamdb/moonstreamdb/blockchain.py index af73e3eb..78c3871f 100644 --- a/moonstreamdb/moonstreamdb/blockchain.py +++ b/moonstreamdb/moonstreamdb/blockchain.py @@ -32,6 +32,9 @@ from .models import ( XaiBlock, XaiLabel, XaiTransaction, + XaiTestnetBlock, + XaiTestnetLabel, + XaiTestnetTransaction, ) @@ -46,6 +49,7 @@ class AvailableBlockchainType(Enum): ARBITRUM_NOVA = "arbitrum_nova" ARBITRUM_SEPOLIA = "arbitrum_sepolia" XAI = "xai" + XAI_TESTNET = "xai_testnet" def get_block_model( @@ -62,10 +66,11 @@ def get_block_model( ArbitrumNovaBlock, ArbitrumSepoliaBlock, XaiBlock, + XaiTestnetBlock, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock, ArbitrumSepoliaBlock, XaiBlock + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock, ArbitrumSepoliaBlock, XaiBlock, XaiTestnetBlock set proper blocks model. """ block_model: Type[ @@ -80,6 +85,7 @@ def get_block_model( ArbitrumNovaBlock, ArbitrumSepoliaBlock, XaiBlock, + XaiTestnetBlock, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -102,6 +108,8 @@ def get_block_model( block_model = ArbitrumSepoliaBlock elif blockchain_type == AvailableBlockchainType.XAI: block_model = XaiBlock + elif blockchain_type == AvailableBlockchainType.XAI_TESTNET: + block_model = XaiTestnetBlock else: raise Exception("Unsupported blockchain type provided") @@ -122,10 +130,11 @@ def get_label_model( ArbitrumNovaLabel, ArbitrumSepoliaLabel, XaiLabel, + XaiTestnetLabel, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel, ArbitrumSepoliaLabel, XaiLabel + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel, ArbitrumSepoliaLabel, XaiLabel, XaiTestnetLabel set proper block label model. """ label_model: Type[ @@ -140,6 +149,7 @@ def get_label_model( ArbitrumNovaLabel, ArbitrumSepoliaLabel, XaiLabel, + XaiTestnetLabel, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -162,6 +172,8 @@ def get_label_model( label_model = ArbitrumSepoliaLabel elif blockchain_type == AvailableBlockchainType.XAI: label_model = XaiLabel + elif blockchain_type == AvailableBlockchainType.XAI_TESTNET: + label_model = XaiTestnetLabel else: raise Exception("Unsupported blockchain type provided") @@ -182,10 +194,11 @@ def get_transaction_model( ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, XaiTransaction, + XaiTestnetTransaction, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, XaiTransaction + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, XaiTransaction, XaiTestnetTransaction set proper block transactions model. """ transaction_model: Type[ @@ -200,6 +213,7 @@ def get_transaction_model( ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, XaiTransaction, + XaiTestnetTransaction, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -222,6 +236,8 @@ def get_transaction_model( transaction_model = ArbitrumSepoliaTransaction elif blockchain_type == AvailableBlockchainType.XAI: transaction_model = XaiTransaction + elif blockchain_type == AvailableBlockchainType.XAI_TESTNET: + transaction_model = XaiTestnetTransaction else: raise Exception("Unsupported blockchain type provided") diff --git a/moonstreamdb/moonstreamdb/models.py b/moonstreamdb/moonstreamdb/models.py index df223d35..50ec63e8 100644 --- a/moonstreamdb/moonstreamdb/models.py +++ b/moonstreamdb/moonstreamdb/models.py @@ -1101,6 +1101,106 @@ class XaiLabel(Base): # type: ignore ) +class XaiTestnetBlock(Base): # type: ignore + __tablename__ = "xai_testnet_blocks" + + block_number = Column( + BigInteger, primary_key=True, unique=True, nullable=False, index=True + ) + difficulty = Column(BigInteger) + extra_data = Column(VARCHAR(128)) + gas_limit = Column(BigInteger) + gas_used = Column(BigInteger) + base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + hash = Column(VARCHAR(256), index=True) + logs_bloom = Column(VARCHAR(1024)) + miner = Column(VARCHAR(256)) + nonce = Column(VARCHAR(256)) + parent_hash = Column(VARCHAR(256)) + receipt_root = Column(VARCHAR(256)) + uncles = Column(VARCHAR(256)) + size = Column(Integer) + state_root = Column(VARCHAR(256)) + timestamp = Column(BigInteger, index=True) + total_difficulty = Column(VARCHAR(256)) + transactions_root = Column(VARCHAR(256)) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + sha3_uncles = Column(VARCHAR(256), nullable=True) + l1_block_number = Column(BigInteger, nullable=True) + send_count = Column(BigInteger, nullable=True) + send_root = Column(VARCHAR(256), nullable=True) + mix_hash = Column(VARCHAR(256), nullable=True) + + +class XaiTestnetTransaction(Base): # type: ignore + __tablename__ = "xai_testnet_transactions" + + hash = Column( + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True + ) + block_number = Column( + BigInteger, + ForeignKey("xai_testnet_blocks.block_number", ondelete="CASCADE"), + nullable=False, + index=True, + ) + from_address = Column(VARCHAR(256), index=True) + to_address = Column(VARCHAR(256), index=True) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) + max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + input = Column(Text) + nonce = Column(VARCHAR(256)) + transaction_index = Column(BigInteger) + transaction_type = Column(Integer, nullable=True) + value = Column(Numeric(precision=78, scale=0), index=True) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + y_parity = Column(BigInteger, nullable=True) + + +class XaiTestnetLabel(Base): # type: ignore + __tablename__ = "xai_testnet_labels" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + nullable=False, + ) + label = Column(VARCHAR(256), nullable=False, index=True) + block_number = Column( + BigInteger, + nullable=True, + index=True, + ) + address = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + transaction_hash = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + label_data = Column(JSONB, nullable=True) + block_timestamp = Column(BigInteger, index=True) + log_index = Column(Integer, nullable=True) + created_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + class ESDFunctionSignature(Base): # type: ignore """ Function signature from blockchain (Ethereum/Polygon) Signature Database. diff --git a/moonstreamdb/moonstreamdb/networks.py b/moonstreamdb/moonstreamdb/networks.py index 30256ca5..3873fbc8 100644 --- a/moonstreamdb/moonstreamdb/networks.py +++ b/moonstreamdb/moonstreamdb/networks.py @@ -33,6 +33,9 @@ from .models import ( XaiBlock, XaiLabel, XaiTransaction, + XaiTestnetBlock, + XaiTestnetLabel, + XaiTestnetTransaction, ) @@ -47,6 +50,7 @@ class Network(Enum): arbitrum_nova = "arbitrum_nova" arbitrum_sepolia = "arbitrum_sepolia" xai = "xai" + xai_testnet = "xai_testnet" tx_raw_types = Union[ @@ -60,6 +64,7 @@ tx_raw_types = Union[ ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, XaiTransaction, + XaiTestnetTransaction, ] MODELS: Dict[Network, Dict[str, Base]] = { @@ -113,4 +118,9 @@ MODELS: Dict[Network, Dict[str, Base]] = { "labels": XaiLabel, "transactions": XaiTransaction, }, + Network.xai_testnet: { + "blocks": XaiTestnetBlock, + "labels": XaiTestnetLabel, + "transactions": XaiTestnetTransaction, + }, } diff --git a/moonstreamdb/moonstreamdb/version.py b/moonstreamdb/moonstreamdb/version.py index 73a7e3a0..0facaebf 100644 --- a/moonstreamdb/moonstreamdb/version.py +++ b/moonstreamdb/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.3.8" +MOONSTREAMDB_VERSION = "0.3.9"